srclawyer 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -2
- package/dist/cli.js +11 -1
- package/dist/cloud/protectedLogicClient.js +30 -6
- package/dist/commands/init.d.ts +5 -0
- package/dist/commands/init.js +36 -11
- package/dist/commands/scan.js +9 -3
- package/dist/engine/discoverFiles.d.ts +2 -1
- package/dist/engine/discoverFiles.js +7 -7
- package/dist/engine/discoverProjects.d.ts +9 -1
- package/dist/engine/discoverProjects.js +71 -18
- package/dist/engine/globFiles.d.ts +10 -0
- package/dist/engine/globFiles.js +12 -0
- package/dist/engine/languageDetection.d.ts +2 -1
- package/dist/engine/languageDetection.js +10 -10
- package/dist/engine/rootBoundary.d.ts +29 -0
- package/dist/engine/rootBoundary.js +102 -0
- package/dist/engine/scanEngine.js +174 -58
- package/dist/engine/scanLimits.d.ts +29 -0
- package/dist/engine/scanLimits.js +70 -0
- package/dist/engine/tsProject.d.ts +2 -1
- package/dist/engine/tsProject.js +40 -3
- package/dist/engine/types.d.ts +16 -0
- package/dist/engine/workspacePatterns.d.ts +23 -0
- package/dist/engine/workspacePatterns.js +140 -0
- package/dist/llm/buildRequestItem.d.ts +5 -1
- package/dist/llm/buildRequestItem.js +6 -2
- package/dist/llm/resolveAmbiguousFindings.js +7 -3
- package/dist/llm/resolveProviderForTier.js +1 -1
- package/dist/policy/generatePolicy.js +33 -8
- package/dist/policy/legalNotices.js +11 -7
- package/dist/report/formatText.d.ts +6 -1
- package/dist/report/formatText.js +23 -2
- package/dist/rules/htmlRules.d.ts +11 -0
- package/dist/rules/htmlRules.js +73 -17
- package/dist/rules/zodPreFilter.js +10 -1
- package/dist/schemaParsers/openapi.d.ts +6 -1
- package/dist/schemaParsers/openapi.js +3 -1
- package/package.json +2 -2
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { realpath, realpathSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
// Directories whose symlinks are never scanned anyway (they are globbed with ignore patterns); a
|
|
4
|
+
// symlink out of one of these is blocked as usual but not worth a warning -- e.g. a package manager
|
|
5
|
+
// pointing node_modules entries at a shared store outside the repo.
|
|
6
|
+
const QUIET_SEGMENTS = new Set(["node_modules", ".git"]);
|
|
7
|
+
/**
|
|
8
|
+
* The scanned root as a security boundary: a scanned repo is untrusted input, so no file whose REAL
|
|
9
|
+
* path resolves outside it may be read, and no directory symlink leaving it may be walked (a symlink
|
|
10
|
+
* to "/" or "~" would otherwise make a scan traverse the whole filesystem and report lines from files
|
|
11
|
+
* the repo's author chose). In-root symlinks, including loops that stay inside, keep working.
|
|
12
|
+
*/
|
|
13
|
+
export class RootBoundary {
|
|
14
|
+
root;
|
|
15
|
+
realRoot;
|
|
16
|
+
// canonical in-repo location of the link entry -> its shortest lexical path. A link reached again
|
|
17
|
+
// through an in-root directory loop is the same link, counted once.
|
|
18
|
+
blocked = new Map();
|
|
19
|
+
resolved = new Map();
|
|
20
|
+
constructor(root) {
|
|
21
|
+
this.root = root;
|
|
22
|
+
try {
|
|
23
|
+
this.realRoot = realpathSync(root);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
this.realRoot = resolve(root);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
isInsideRealPath(resolved) {
|
|
30
|
+
return resolved === this.realRoot || resolved.startsWith(this.realRoot + sep);
|
|
31
|
+
}
|
|
32
|
+
/** Records a blocked link by its in-repo (lexical) path only -- never its target. */
|
|
33
|
+
recordBlocked(linkPath) {
|
|
34
|
+
const rel = relative(this.root, linkPath);
|
|
35
|
+
if (rel === "" || rel.startsWith(".."))
|
|
36
|
+
return;
|
|
37
|
+
if (rel.split(sep).some((segment) => QUIET_SEGMENTS.has(segment)))
|
|
38
|
+
return;
|
|
39
|
+
let canonical = rel;
|
|
40
|
+
try {
|
|
41
|
+
canonical = join(realpathSync(dirname(linkPath)), basename(linkPath));
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// keep the lexical path
|
|
45
|
+
}
|
|
46
|
+
const existing = this.blocked.get(canonical);
|
|
47
|
+
if (existing === undefined || rel.length < existing.length)
|
|
48
|
+
this.blocked.set(canonical, rel);
|
|
49
|
+
}
|
|
50
|
+
/** Read-time check, independent of how the path was discovered: the real path if it is inside the
|
|
51
|
+
* root, else null (and an out-of-root link is recorded). Unresolvable paths fail closed. Cached. */
|
|
52
|
+
resolveInside(path) {
|
|
53
|
+
if (this.resolved.has(path))
|
|
54
|
+
return this.resolved.get(path);
|
|
55
|
+
let result = null;
|
|
56
|
+
try {
|
|
57
|
+
const real = realpathSync(path);
|
|
58
|
+
if (this.isInsideRealPath(real))
|
|
59
|
+
result = real;
|
|
60
|
+
else
|
|
61
|
+
this.recordBlocked(path);
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
result = null;
|
|
65
|
+
}
|
|
66
|
+
this.resolved.set(path, result);
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
69
|
+
allowsPath(path) {
|
|
70
|
+
return this.resolveInside(path) !== null;
|
|
71
|
+
}
|
|
72
|
+
/** `fs` overrides for tinyglobby: a symlink resolving outside the root errors out, so it is neither
|
|
73
|
+
* walked (directories) nor returned (files). */
|
|
74
|
+
globFs() {
|
|
75
|
+
const checked = (path, resolved) => {
|
|
76
|
+
if (this.isInsideRealPath(resolved))
|
|
77
|
+
return null;
|
|
78
|
+
this.recordBlocked(path);
|
|
79
|
+
return "outside-root";
|
|
80
|
+
};
|
|
81
|
+
const asyncRealpath = ((path, callback) => {
|
|
82
|
+
realpath(path, (err, resolved) => {
|
|
83
|
+
if (err)
|
|
84
|
+
return callback(err);
|
|
85
|
+
const problem = checked(path, resolved);
|
|
86
|
+
callback(problem ? Object.assign(new Error(problem), { code: "EXDEV" }) : null, problem ? undefined : resolved);
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
const syncRealpath = ((path) => {
|
|
90
|
+
const resolved = realpathSync(path);
|
|
91
|
+
const problem = checked(path, resolved);
|
|
92
|
+
if (problem)
|
|
93
|
+
throw Object.assign(new Error(problem), { code: "EXDEV" });
|
|
94
|
+
return resolved;
|
|
95
|
+
});
|
|
96
|
+
return { realpath: asyncRealpath, realpathSync: syncRealpath };
|
|
97
|
+
}
|
|
98
|
+
/** Relative in-repo paths of every blocked link, sorted; call after discovery and reading are done. */
|
|
99
|
+
blockedLinks() {
|
|
100
|
+
return [...this.blocked.values()].sort();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
|
-
import { extname } from "node:path";
|
|
2
|
+
import { extname, relative } from "node:path";
|
|
3
3
|
import { discoverProjects } from "./discoverProjects.js";
|
|
4
4
|
import { discoverFiles } from "./discoverFiles.js";
|
|
5
5
|
import { detectUnsupportedStack } from "./languageDetection.js";
|
|
@@ -13,7 +13,9 @@ import { runPythonSdkRules } from "../rules/pythonSdkRules.js";
|
|
|
13
13
|
import { runPythonFrameworkRules } from "../rules/pythonFrameworkRules.js";
|
|
14
14
|
import { findZodCandidates } from "../rules/zodPreFilter.js";
|
|
15
15
|
import { buildTsProject } from "./tsProject.js";
|
|
16
|
-
import { runHtmlInputRules,
|
|
16
|
+
import { runHtmlInputRules, extractInlineScriptsBounded } from "../rules/htmlRules.js";
|
|
17
|
+
import { containsTemplateSyntax, isOversizedFile, summarizeSkippedFiles, summarizeTemplateScripts, } from "./scanLimits.js";
|
|
18
|
+
import { RootBoundary } from "./rootBoundary.js";
|
|
17
19
|
import { parseOpenApiSpec } from "../schemaParsers/openapi.js";
|
|
18
20
|
import { parseGraphQLSchema } from "../schemaParsers/graphql.js";
|
|
19
21
|
const OPENAPI_NAME = /(openapi|swagger)\.(ya?ml|json)$/i;
|
|
@@ -23,12 +25,12 @@ const GRAPHQL_NAME = /\.(graphql|gql)$/i;
|
|
|
23
25
|
// ~2.5s/~800MB and ~2.2s/~650MB respectively for a full ts.Program build. Eager construction would tax
|
|
24
26
|
// every scan of a large project regardless of whether it ever hits the one pattern that needs it;
|
|
25
27
|
// memoized-lazy pays that cost at most once, only on the scan that actually earns it.
|
|
26
|
-
function makeLazyTsProject(files) {
|
|
28
|
+
function makeLazyTsProject(files, boundary) {
|
|
27
29
|
let handle;
|
|
28
30
|
return () => {
|
|
29
31
|
if (handle === undefined) {
|
|
30
32
|
try {
|
|
31
|
-
handle = buildTsProject(files);
|
|
33
|
+
handle = buildTsProject(files, boundary);
|
|
32
34
|
}
|
|
33
35
|
catch {
|
|
34
36
|
handle = null;
|
|
@@ -37,25 +39,69 @@ function makeLazyTsProject(files) {
|
|
|
37
39
|
return handle;
|
|
38
40
|
};
|
|
39
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Per-file containment: nothing a single file does (a parse failure, or an exception such as the
|
|
44
|
+
* RangeError a very deep AST throws inside @babel/traverse) may abort the scan or discard the other
|
|
45
|
+
* files' findings. Each rule group is isolated so one group failing keeps the others' findings for the
|
|
46
|
+
* same file, and the failure comes back as a reason for the caller to record -- never swallowed.
|
|
47
|
+
*/
|
|
40
48
|
function runCodeRules(filePath, source, root, getTsProject, zodCandidates, oversizedZodFiles) {
|
|
41
|
-
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
49
|
+
let failure = null;
|
|
50
|
+
const findings = [];
|
|
51
|
+
let ast = null;
|
|
52
|
+
try {
|
|
53
|
+
ast = parseSource(source, filePath);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
ast = null;
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
const zodResult = findZodCandidates(source, filePath, root);
|
|
60
|
+
zodCandidates.push(...zodResult.candidates);
|
|
61
|
+
oversizedZodFiles.push(...zodResult.oversizedFiles);
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
failure = "analysis-failed";
|
|
65
|
+
}
|
|
45
66
|
if (!ast)
|
|
46
|
-
return
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
67
|
+
return { findings, failure: failure ?? "parse-failed" };
|
|
68
|
+
const groups = [
|
|
69
|
+
() => runSdkRules(ast, filePath, source, root),
|
|
70
|
+
() => runFrameworkRules(ast, filePath, source, root),
|
|
71
|
+
() => runWebApiRules(ast, filePath, source, root),
|
|
72
|
+
() => runTypedRequestBodyRules(ast, filePath, source, root, getTsProject),
|
|
52
73
|
];
|
|
74
|
+
for (const run of groups) {
|
|
75
|
+
try {
|
|
76
|
+
findings.push(...run());
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
failure = "analysis-failed";
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return { findings, failure };
|
|
53
83
|
}
|
|
54
84
|
async function runPythonCodeRules(filePath, source, root) {
|
|
55
|
-
|
|
85
|
+
let tree = null;
|
|
86
|
+
try {
|
|
87
|
+
tree = await parsePythonSource(source);
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
tree = null;
|
|
91
|
+
}
|
|
56
92
|
if (!tree)
|
|
57
|
-
return [];
|
|
58
|
-
|
|
93
|
+
return { findings: [], failure: "parse-failed" };
|
|
94
|
+
const findings = [];
|
|
95
|
+
let failure = null;
|
|
96
|
+
for (const run of [() => runPythonSdkRules(tree, filePath, source, root), () => runPythonFrameworkRules(tree, filePath, source, root)]) {
|
|
97
|
+
try {
|
|
98
|
+
findings.push(...run());
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
failure = "analysis-failed";
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return { findings, failure };
|
|
59
105
|
}
|
|
60
106
|
/**
|
|
61
107
|
* Pure, local, offline — no network access anywhere in this function or anything it calls. Zod
|
|
@@ -67,13 +113,60 @@ async function runPythonCodeRules(filePath, source, root) {
|
|
|
67
113
|
* reading this file alone.
|
|
68
114
|
*/
|
|
69
115
|
export async function scan(root) {
|
|
70
|
-
const
|
|
116
|
+
const boundary = new RootBoundary(root);
|
|
117
|
+
const [{ projects, workspaceWarning }, unsupportedStackWarning] = await Promise.all([
|
|
118
|
+
discoverProjects(root, boundary),
|
|
119
|
+
detectUnsupportedStack(root, boundary),
|
|
120
|
+
]);
|
|
71
121
|
const findings = [];
|
|
72
122
|
const pendingZodCandidates = [];
|
|
73
123
|
const oversizedZodFiles = [];
|
|
124
|
+
const skippedFiles = [];
|
|
74
125
|
let filesScanned = 0;
|
|
126
|
+
const recordSkip = (filePath, reason) => {
|
|
127
|
+
const file = relative(root, filePath);
|
|
128
|
+
if (!skippedFiles.some((s) => s.file === file && s.reason === reason))
|
|
129
|
+
skippedFiles.push({ file, reason });
|
|
130
|
+
};
|
|
131
|
+
// The one gate every scanned file passes through before any content is read: real path inside the
|
|
132
|
+
// scanned root (the glob layer already refuses to return out-of-root symlinks; this re-checks at read
|
|
133
|
+
// time), then the shared size cap, then the read itself. A skipped file is recorded and surfaced as
|
|
134
|
+
// a warning, never silently dropped. Returns null when the file must not be analyzed.
|
|
135
|
+
const readScannable = (filePath) => {
|
|
136
|
+
if (!boundary.allowsPath(filePath))
|
|
137
|
+
return null; // recorded by the boundary as a blocked link
|
|
138
|
+
if (isOversizedFile(filePath)) {
|
|
139
|
+
recordSkip(filePath, "oversized");
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
filesScanned += 1;
|
|
143
|
+
try {
|
|
144
|
+
return readFileSync(filePath, "utf8");
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
recordSkip(filePath, "unreadable");
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
};
|
|
75
151
|
const projectRootList = [...new Set(projects.map((p) => p.root))];
|
|
76
|
-
const fileSets = await Promise.all(projectRootList.map((projectRoot) => discoverFiles(projectRoot)));
|
|
152
|
+
const fileSets = await Promise.all(projectRootList.map((projectRoot) => discoverFiles(projectRoot, boundary)));
|
|
153
|
+
// The same real file can be reached by several paths (an in-root symlink, or a directory loop that
|
|
154
|
+
// stays inside the root); scan it once, under its shortest path, so findings aren't duplicated.
|
|
155
|
+
const seenRealPaths = new Map();
|
|
156
|
+
const claimsRealPath = (filePath) => {
|
|
157
|
+
const real = boundary.resolveInside(filePath);
|
|
158
|
+
if (real === null)
|
|
159
|
+
return true; // let readScannable record why it is being skipped
|
|
160
|
+
const holder = seenRealPaths.get(real);
|
|
161
|
+
if (holder === undefined) {
|
|
162
|
+
seenRealPaths.set(real, filePath);
|
|
163
|
+
return true;
|
|
164
|
+
}
|
|
165
|
+
return holder === filePath;
|
|
166
|
+
};
|
|
167
|
+
const shortestPathFirst = (paths) => [...paths].sort((a, b) => a.length - b.length || a.localeCompare(b));
|
|
168
|
+
// Claimed shortest-first, but returned in the original sorted order so report order is unchanged.
|
|
169
|
+
const claimed = (paths) => shortestPathFirst(paths).filter(claimsRealPath).sort();
|
|
77
170
|
const allCodeFiles = new Set();
|
|
78
171
|
const allPythonFiles = new Set();
|
|
79
172
|
const allHtmlFiles = new Set();
|
|
@@ -82,74 +175,93 @@ export async function scan(root) {
|
|
|
82
175
|
const tsProjectByRoot = new Map();
|
|
83
176
|
fileSets.forEach((set, i) => {
|
|
84
177
|
const projectRoot = projectRootList[i];
|
|
85
|
-
tsProjectByRoot.set(projectRoot, makeLazyTsProject(set.codeFiles));
|
|
86
|
-
set.codeFiles.forEach((f) => {
|
|
178
|
+
tsProjectByRoot.set(projectRoot, makeLazyTsProject(set.codeFiles, boundary));
|
|
179
|
+
claimed(set.codeFiles).forEach((f) => {
|
|
87
180
|
allCodeFiles.add(f);
|
|
88
181
|
rootForFile.set(f, projectRoot);
|
|
89
182
|
});
|
|
90
|
-
set.pythonFiles.forEach((f) => allPythonFiles.add(f));
|
|
91
|
-
set.htmlFiles.forEach((f) => allHtmlFiles.add(f));
|
|
92
|
-
set.schemaFiles.forEach((f) => allSchemaFiles.add(f));
|
|
183
|
+
claimed(set.pythonFiles).forEach((f) => allPythonFiles.add(f));
|
|
184
|
+
claimed(set.htmlFiles).forEach((f) => allHtmlFiles.add(f));
|
|
185
|
+
claimed(set.schemaFiles).forEach((f) => allSchemaFiles.add(f));
|
|
93
186
|
});
|
|
94
187
|
// HTML files (and their inline <script> blocks) aren't part of any codeFiles list and never carry
|
|
95
188
|
// a resolvable TS type, so they get a permanently-null thunk rather than a project association.
|
|
96
189
|
const noTsProject = () => null;
|
|
97
190
|
for (const filePath of allCodeFiles) {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
try {
|
|
101
|
-
source = readFileSync(filePath, "utf8");
|
|
102
|
-
}
|
|
103
|
-
catch {
|
|
191
|
+
const source = readScannable(filePath);
|
|
192
|
+
if (source === null)
|
|
104
193
|
continue;
|
|
105
|
-
}
|
|
106
194
|
const getTsProject = tsProjectByRoot.get(rootForFile.get(filePath) ?? "") ?? noTsProject;
|
|
107
|
-
|
|
195
|
+
const result = runCodeRules(filePath, source, root, getTsProject, pendingZodCandidates, oversizedZodFiles);
|
|
196
|
+
findings.push(...result.findings);
|
|
197
|
+
if (result.failure)
|
|
198
|
+
recordSkip(filePath, result.failure);
|
|
108
199
|
}
|
|
109
200
|
for (const filePath of allPythonFiles) {
|
|
110
|
-
|
|
111
|
-
|
|
201
|
+
const source = readScannable(filePath);
|
|
202
|
+
if (source === null)
|
|
203
|
+
continue;
|
|
204
|
+
const result = await runPythonCodeRules(filePath, source, root);
|
|
205
|
+
findings.push(...result.findings);
|
|
206
|
+
if (result.failure)
|
|
207
|
+
recordSkip(filePath, result.failure);
|
|
208
|
+
}
|
|
209
|
+
for (const filePath of allHtmlFiles) {
|
|
210
|
+
const html = readScannable(filePath);
|
|
211
|
+
if (html === null)
|
|
212
|
+
continue;
|
|
112
213
|
try {
|
|
113
|
-
|
|
214
|
+
findings.push(...runHtmlInputRules(html, filePath, root));
|
|
114
215
|
}
|
|
115
216
|
catch {
|
|
116
|
-
|
|
217
|
+
recordSkip(filePath, "analysis-failed");
|
|
117
218
|
}
|
|
118
|
-
|
|
119
|
-
}
|
|
120
|
-
for (const filePath of allHtmlFiles) {
|
|
121
|
-
filesScanned += 1;
|
|
122
|
-
let html;
|
|
219
|
+
let extraction = { scripts: [], skippedCount: 0 };
|
|
123
220
|
try {
|
|
124
|
-
|
|
221
|
+
extraction = extractInlineScriptsBounded(html);
|
|
125
222
|
}
|
|
126
223
|
catch {
|
|
127
|
-
|
|
224
|
+
recordSkip(filePath, "analysis-failed");
|
|
128
225
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
226
|
+
if (extraction.skippedCount > 0)
|
|
227
|
+
recordSkip(filePath, "inline-scripts-truncated");
|
|
228
|
+
let templateScripts = 0;
|
|
229
|
+
for (const script of extraction.scripts) {
|
|
230
|
+
const result = runCodeRules(filePath, script.code, root, noTsProject, pendingZodCandidates, oversizedZodFiles);
|
|
231
|
+
findings.push(...result.findings);
|
|
232
|
+
if (result.failure === "parse-failed" && containsTemplateSyntax(script.code))
|
|
233
|
+
templateScripts += 1;
|
|
234
|
+
else if (result.failure)
|
|
235
|
+
recordSkip(filePath, result.failure);
|
|
132
236
|
}
|
|
237
|
+
if (templateScripts > 0)
|
|
238
|
+
skippedFiles.push({ file: relative(root, filePath), reason: "template-syntax", scripts: templateScripts });
|
|
133
239
|
}
|
|
134
240
|
for (const filePath of allSchemaFiles) {
|
|
241
|
+
if (!boundary.allowsPath(filePath))
|
|
242
|
+
continue;
|
|
243
|
+
if (isOversizedFile(filePath)) {
|
|
244
|
+
recordSkip(filePath, "oversized");
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
135
247
|
filesScanned += 1;
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
248
|
+
try {
|
|
249
|
+
if (OPENAPI_NAME.test(filePath)) {
|
|
250
|
+
const report = { hadErrors: false };
|
|
251
|
+
findings.push(...parseOpenApiSpec(filePath, root, report));
|
|
252
|
+
if (report.hadErrors)
|
|
253
|
+
recordSkip(filePath, "parse-failed");
|
|
142
254
|
}
|
|
143
|
-
|
|
144
|
-
else if (GRAPHQL_NAME.test(filePath) || extname(filePath) === ".graphql") {
|
|
145
|
-
try {
|
|
255
|
+
else if (GRAPHQL_NAME.test(filePath) || extname(filePath) === ".graphql") {
|
|
146
256
|
findings.push(...parseGraphQLSchema(filePath, root));
|
|
147
257
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
recordSkip(filePath, "parse-failed");
|
|
151
261
|
}
|
|
152
262
|
}
|
|
263
|
+
for (const link of boundary.blockedLinks())
|
|
264
|
+
skippedFiles.push({ file: link, reason: "symlink-outside-root" });
|
|
153
265
|
const ambiguousCount = findings.filter((f) => f.requiresReview).length;
|
|
154
266
|
const maxZodFragmentBytes = pendingZodCandidates.length
|
|
155
267
|
? Math.max(...pendingZodCandidates.map((c) => Buffer.byteLength(c.codeFragment, "utf8")))
|
|
@@ -160,6 +272,10 @@ export async function scan(root) {
|
|
|
160
272
|
ambiguousCount,
|
|
161
273
|
filesScanned,
|
|
162
274
|
unsupportedStackWarning,
|
|
275
|
+
workspaceWarning,
|
|
276
|
+
skippedFiles,
|
|
277
|
+
skippedFilesWarning: summarizeSkippedFiles(skippedFiles),
|
|
278
|
+
templateSyntaxNote: summarizeTemplateScripts(skippedFiles),
|
|
163
279
|
pendingZodCandidates,
|
|
164
280
|
oversizedZodFiles,
|
|
165
281
|
maxZodFragmentBytes,
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One cap for every scanned file type (code, Python, HTML, OpenAPI/GraphQL). The largest real file in
|
|
3
|
+
* the golden corpus is a 408KB OpenAPI spec, so 2 MiB leaves ~5x headroom while still bounding the
|
|
4
|
+
* parse/regex/memory cost a single hostile file can impose. Files over it are skipped WITH a warning
|
|
5
|
+
* (see summarizeSkippedFiles), never silently.
|
|
6
|
+
*/
|
|
7
|
+
export declare const MAX_SCANNED_FILE_BYTES: number;
|
|
8
|
+
export type SkipReason = "oversized" | "inline-scripts-truncated" | "symlink-outside-root" | "unreadable" | "parse-failed" | "analysis-failed" | "template-syntax";
|
|
9
|
+
export interface SkippedFile {
|
|
10
|
+
file: string;
|
|
11
|
+
reason: SkipReason;
|
|
12
|
+
/** For "template-syntax" only: how many inline scripts in the file were not analyzed. */
|
|
13
|
+
scripts?: number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Informational reasons: the content was deliberately not analyzed because it isn't JavaScript yet
|
|
17
|
+
* (an inline script still carrying Jinja/ERB/Handlebars delimiters), which is expected for template
|
|
18
|
+
* files rather than a sign the scan is incomplete. They are listed in --json and noted in one line,
|
|
19
|
+
* but get no policy banner and no non-zero exit code. Every other reason is a real gap.
|
|
20
|
+
*/
|
|
21
|
+
export declare function isInformationalSkip(reason: SkipReason): boolean;
|
|
22
|
+
export declare function containsTemplateSyntax(code: string): boolean;
|
|
23
|
+
/** One informational line, or null when no template-syntax scripts were skipped. */
|
|
24
|
+
export declare function summarizeTemplateScripts(skipped: SkippedFile[]): string | null;
|
|
25
|
+
/** False for files that can't be stat'ed -- the existing read-failure handling covers those. */
|
|
26
|
+
export declare function isOversizedFile(path: string, maxBytes?: number): boolean;
|
|
27
|
+
/** A single summarized line -- a few names, the rest only in ScanResult.skippedFiles (--json). Names
|
|
28
|
+
* are in-repo paths only; a blocked symlink's target is never recorded or printed. */
|
|
29
|
+
export declare function summarizeSkippedFiles(allSkipped: SkippedFile[]): string | null;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { statSync } from "node:fs";
|
|
2
|
+
/**
|
|
3
|
+
* One cap for every scanned file type (code, Python, HTML, OpenAPI/GraphQL). The largest real file in
|
|
4
|
+
* the golden corpus is a 408KB OpenAPI spec, so 2 MiB leaves ~5x headroom while still bounding the
|
|
5
|
+
* parse/regex/memory cost a single hostile file can impose. Files over it are skipped WITH a warning
|
|
6
|
+
* (see summarizeSkippedFiles), never silently.
|
|
7
|
+
*/
|
|
8
|
+
export const MAX_SCANNED_FILE_BYTES = 2 * 1024 * 1024;
|
|
9
|
+
/**
|
|
10
|
+
* Informational reasons: the content was deliberately not analyzed because it isn't JavaScript yet
|
|
11
|
+
* (an inline script still carrying Jinja/ERB/Handlebars delimiters), which is expected for template
|
|
12
|
+
* files rather than a sign the scan is incomplete. They are listed in --json and noted in one line,
|
|
13
|
+
* but get no policy banner and no non-zero exit code. Every other reason is a real gap.
|
|
14
|
+
*/
|
|
15
|
+
export function isInformationalSkip(reason) {
|
|
16
|
+
return reason === "template-syntax";
|
|
17
|
+
}
|
|
18
|
+
// Content a template engine fills in before the browser ever sees it: Jinja/Django/Twig/Liquid
|
|
19
|
+
// ({{ }}, {% %}, {# #}) and ERB/EJS (<% %>).
|
|
20
|
+
const TEMPLATE_DELIMITERS = /\{\{|\{%|<%|\{#/;
|
|
21
|
+
export function containsTemplateSyntax(code) {
|
|
22
|
+
return TEMPLATE_DELIMITERS.test(code);
|
|
23
|
+
}
|
|
24
|
+
/** One informational line, or null when no template-syntax scripts were skipped. */
|
|
25
|
+
export function summarizeTemplateScripts(skipped) {
|
|
26
|
+
const total = skipped.filter((s) => s.reason === "template-syntax").reduce((n, s) => n + (s.scripts ?? 0), 0);
|
|
27
|
+
if (total === 0)
|
|
28
|
+
return null;
|
|
29
|
+
const subject = total === 1 ? "1 inline script in template files was" : `${total} inline scripts in template files were`;
|
|
30
|
+
return `${subject} not analyzed (listed in --json output).`;
|
|
31
|
+
}
|
|
32
|
+
const REASON_LABELS = {
|
|
33
|
+
oversized: "over the size cap",
|
|
34
|
+
"inline-scripts-truncated": "too many inline scripts to analyze",
|
|
35
|
+
"symlink-outside-root": "symlink(s) resolving outside the scanned root, not followed",
|
|
36
|
+
unreadable: "unreadable",
|
|
37
|
+
"parse-failed": "failed to parse",
|
|
38
|
+
"analysis-failed": "aborted during analysis",
|
|
39
|
+
"template-syntax": "inline scripts with template syntax",
|
|
40
|
+
};
|
|
41
|
+
/** False for files that can't be stat'ed -- the existing read-failure handling covers those. */
|
|
42
|
+
export function isOversizedFile(path, maxBytes = MAX_SCANNED_FILE_BYTES) {
|
|
43
|
+
try {
|
|
44
|
+
return statSync(path).size > maxBytes;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const PREVIEW_COUNT = 3;
|
|
51
|
+
/** A single summarized line -- a few names, the rest only in ScanResult.skippedFiles (--json). Names
|
|
52
|
+
* are in-repo paths only; a blocked symlink's target is never recorded or printed. */
|
|
53
|
+
export function summarizeSkippedFiles(allSkipped) {
|
|
54
|
+
const skipped = allSkipped.filter((s) => !isInformationalSkip(s.reason));
|
|
55
|
+
if (skipped.length === 0)
|
|
56
|
+
return null;
|
|
57
|
+
const names = [...new Set(skipped.map((s) => s.file))];
|
|
58
|
+
const preview = names.slice(0, PREVIEW_COUNT).join(", ");
|
|
59
|
+
const remaining = names.length - PREVIEW_COUNT;
|
|
60
|
+
const suffix = remaining > 0 ? `, and ${remaining} more (full list in --json output)` : "";
|
|
61
|
+
const counts = new Map();
|
|
62
|
+
for (const s of skipped)
|
|
63
|
+
counts.set(s.reason, (counts.get(s.reason) ?? new Set()).add(s.file));
|
|
64
|
+
const breakdown = Object.keys(REASON_LABELS)
|
|
65
|
+
.filter((reason) => counts.has(reason))
|
|
66
|
+
.map((reason) => `${counts.get(reason).size} ${REASON_LABELS[reason]}`)
|
|
67
|
+
.join("; ");
|
|
68
|
+
const limit = `${MAX_SCANNED_FILE_BYTES / (1024 * 1024)} MiB`;
|
|
69
|
+
return `${names.length} file(s) were skipped or only partially scanned (${breakdown.replace("over the size cap", `over the ${limit} size cap`)}): ${preview}${suffix}. Findings in them may be missing, so this scan is incomplete.`;
|
|
70
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
+
import type { RootBoundary } from "./rootBoundary.js";
|
|
1
2
|
export interface TsProjectHandle {
|
|
2
3
|
getObjectTypeMembersAt(filePath: string, line: number, column: number): string[] | null;
|
|
3
4
|
}
|
|
4
|
-
export declare function buildTsProject(files: string[]): TsProjectHandle;
|
|
5
|
+
export declare function buildTsProject(files: string[], boundary?: RootBoundary): TsProjectHandle;
|
package/dist/engine/tsProject.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { realpathSync } from "node:fs";
|
|
2
|
+
import { dirname, sep } from "node:path";
|
|
1
3
|
import ts from "typescript";
|
|
2
4
|
// Purely defensive, not a tuned threshold: the largest repo actually measured (dub, 4211 files) took
|
|
3
5
|
// ~2.5s/~800MB to build. This is ~5x that, so a project this large skips type resolution entirely
|
|
@@ -20,18 +22,53 @@ function findNodeAtPosition(root, position) {
|
|
|
20
22
|
});
|
|
21
23
|
return found ?? (root.getStart() <= position && position < root.getEnd() ? root : undefined);
|
|
22
24
|
}
|
|
23
|
-
|
|
25
|
+
const NODE_MODULES_SEGMENT = `${sep}node_modules${sep}`;
|
|
26
|
+
/**
|
|
27
|
+
* TypeScript follows imports on its own, so the list of files handed to createProgram is not the whole
|
|
28
|
+
* set it reads: an import crossing a symlink that leaves the scanned root would otherwise pull that
|
|
29
|
+
* file's type members into findings. The host refuses any file whose REAL path is outside the root,
|
|
30
|
+
* except TypeScript's own lib files and package type declarations (a real path under node_modules --
|
|
31
|
+
* package managers legitimately keep those outside the repo, e.g. pnpm's store).
|
|
32
|
+
*/
|
|
33
|
+
function restrictHostToBoundary(host, options, boundary) {
|
|
34
|
+
const libDir = dirname(ts.getDefaultLibFilePath(options));
|
|
35
|
+
const verdicts = new Map();
|
|
36
|
+
const allowed = (path) => {
|
|
37
|
+
const cached = verdicts.get(path);
|
|
38
|
+
if (cached !== undefined)
|
|
39
|
+
return cached;
|
|
40
|
+
let verdict;
|
|
41
|
+
try {
|
|
42
|
+
const real = realpathSync(path);
|
|
43
|
+
verdict = boundary.isInsideRealPath(real) || real.startsWith(libDir + sep) || real.includes(NODE_MODULES_SEGMENT);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
verdict = true; // doesn't exist (or can't be resolved): the original host answers "no such file"
|
|
47
|
+
}
|
|
48
|
+
verdicts.set(path, verdict);
|
|
49
|
+
return verdict;
|
|
50
|
+
};
|
|
51
|
+
const { fileExists, readFile, getSourceFile } = host;
|
|
52
|
+
host.fileExists = (path) => allowed(path) && fileExists.call(host, path);
|
|
53
|
+
host.readFile = (path) => (allowed(path) ? readFile.call(host, path) : undefined);
|
|
54
|
+
host.getSourceFile = (path, ...rest) => (allowed(path) ? getSourceFile.call(host, path, ...rest) : undefined);
|
|
55
|
+
}
|
|
56
|
+
export function buildTsProject(files, boundary) {
|
|
24
57
|
if (files.length === 0 || files.length > MAX_PROGRAM_FILES)
|
|
25
58
|
return NULL_HANDLE;
|
|
26
59
|
let program;
|
|
27
60
|
let checker;
|
|
28
61
|
try {
|
|
29
|
-
|
|
62
|
+
const options = {
|
|
30
63
|
allowJs: true,
|
|
31
64
|
checkJs: false,
|
|
32
65
|
noEmit: true,
|
|
33
66
|
skipLibCheck: true,
|
|
34
|
-
}
|
|
67
|
+
};
|
|
68
|
+
const host = ts.createCompilerHost(options);
|
|
69
|
+
if (boundary)
|
|
70
|
+
restrictHostToBoundary(host, options, boundary);
|
|
71
|
+
program = ts.createProgram(files, options, host);
|
|
35
72
|
checker = program.getTypeChecker();
|
|
36
73
|
}
|
|
37
74
|
catch {
|
package/dist/engine/types.d.ts
CHANGED
|
@@ -56,6 +56,22 @@ export interface ScanResult {
|
|
|
56
56
|
ambiguousCount: number;
|
|
57
57
|
filesScanned: number;
|
|
58
58
|
unsupportedStackWarning: string | null;
|
|
59
|
+
/** Non-null when the scanned repo's workspace config was rejected and the default package
|
|
60
|
+
* locations were scanned instead (see discoverProjects.ts) -- the scan may be incomplete. */
|
|
61
|
+
workspaceWarning: string | null;
|
|
62
|
+
/** Every file skipped, blocked (out-of-root symlink), unparseable, or aborted mid-analysis, or only
|
|
63
|
+
* partially scanned -- the complete list, including everything the one-line warning below elides. */
|
|
64
|
+
skippedFiles: Array<{
|
|
65
|
+
file: string;
|
|
66
|
+
reason: "oversized" | "inline-scripts-truncated" | "symlink-outside-root" | "unreadable" | "parse-failed" | "analysis-failed" | "template-syntax";
|
|
67
|
+
/** "template-syntax" only: inline scripts in the file that were not analyzed. */
|
|
68
|
+
scripts?: number;
|
|
69
|
+
}>;
|
|
70
|
+
/** One summarized line naming a few of skippedFiles; null when nothing was skipped. */
|
|
71
|
+
skippedFilesWarning: string | null;
|
|
72
|
+
/** One informational line when inline scripts carrying template syntax ({{, {%, <%, {#) were not
|
|
73
|
+
* analyzed; deliberately NOT part of skippedFilesWarning -- no banner, no non-zero exit. */
|
|
74
|
+
templateSyntaxNote: string | null;
|
|
59
75
|
/**
|
|
60
76
|
* Files that look like they might define a Zod schema used to validate incoming request data —
|
|
61
77
|
* unconfirmed, since the actual schema/anchor detection is protected logic and runs server-side.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validation for workspace glob patterns read from a SCANNED repo's own package.json -- i.e. input an
|
|
3
|
+
* attacker controls whenever the repo being scanned isn't the user's own. Brace expansion in glob
|
|
4
|
+
* libraries can be driven to stack overflow, memory exhaustion, or minutes of CPU by very short
|
|
5
|
+
* patterns (a 37-character `{1..300}{1..300}{1..300}` is enough), so patterns are bounded here before
|
|
6
|
+
* they ever reach a glob library. Real workspace patterns (`packages/*`, `{apps,packages}/*`,
|
|
7
|
+
* `tools/*\/packages/*`) are tens of characters with at most one small brace group.
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAX_WORKSPACE_PATTERNS = 100;
|
|
10
|
+
export declare const MAX_PATTERN_LENGTH = 200;
|
|
11
|
+
export declare const MAX_BRACE_DEPTH = 2;
|
|
12
|
+
export declare const MAX_BRACE_GROUPS = 3;
|
|
13
|
+
export declare const MAX_BRACE_EXPANSION = 64;
|
|
14
|
+
export declare const MAX_RANGE_SPAN = 100;
|
|
15
|
+
export type WorkspacePatternsResult = {
|
|
16
|
+
ok: true;
|
|
17
|
+
patterns: string[];
|
|
18
|
+
} | {
|
|
19
|
+
ok: false;
|
|
20
|
+
reason: string;
|
|
21
|
+
};
|
|
22
|
+
/** `raw` is whatever package.json's `workspaces` (or `workspaces.packages`) held, unvalidated. */
|
|
23
|
+
export declare function validateWorkspacePatterns(raw: unknown): WorkspacePatternsResult;
|