srclawyer 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -6
- package/dist/cloud/protectedLogicClient.d.ts +3 -0
- package/dist/cloud/protectedLogicClient.js +62 -20
- package/dist/cloud/resolveProtectedLogic.d.ts +1 -1
- package/dist/cloud/resolveProtectedLogic.js +15 -1
- package/dist/commands/init.js +14 -2
- package/dist/commands/scan.js +25 -7
- package/dist/config/config.d.ts +13 -1
- package/dist/config/config.js +19 -1
- package/dist/engine/discoverFiles.d.ts +1 -0
- package/dist/engine/discoverFiles.js +16 -2
- package/dist/engine/languageDetection.js +37 -7
- package/dist/engine/pythonAstUtils.d.ts +2 -0
- package/dist/engine/pythonAstUtils.js +44 -0
- package/dist/engine/scanEngine.js +42 -5
- package/dist/engine/types.d.ts +9 -0
- package/dist/grammars/LICENSE +21 -0
- package/dist/grammars/tree-sitter-python.wasm +0 -0
- package/dist/llm/resolveProviderForTier.d.ts +1 -1
- package/dist/llm/resolveProviderForTier.js +3 -2
- package/dist/policy/generatePolicy.d.ts +2 -2
- package/dist/policy/generatePolicy.js +15 -2
- package/dist/policy/legalNotices.d.ts +5 -0
- package/dist/policy/legalNotices.js +29 -0
- package/dist/report/formatText.js +1 -1
- package/dist/rules/dataCategoryLexicon.js +9 -3
- package/dist/rules/frameworkRules.js +89 -12
- package/dist/rules/pythonFrameworkRules.d.ts +3 -0
- package/dist/rules/pythonFrameworkRules.js +136 -0
- package/dist/rules/pythonSdkRules.d.ts +3 -0
- package/dist/rules/pythonSdkRules.js +73 -0
- package/dist/rules/safeFieldNames.js +14 -0
- package/dist/rules/zodPreFilter.d.ts +21 -1
- package/dist/rules/zodPreFilter.js +104 -13
- package/package.json +3 -2
|
@@ -4,10 +4,13 @@ import { discoverProjects } from "./discoverProjects.js";
|
|
|
4
4
|
import { discoverFiles } from "./discoverFiles.js";
|
|
5
5
|
import { detectUnsupportedStack } from "./languageDetection.js";
|
|
6
6
|
import { parseSource } from "./astUtils.js";
|
|
7
|
+
import { parsePythonSource } from "./pythonAstUtils.js";
|
|
7
8
|
import { runSdkRules } from "../rules/sdkRules.js";
|
|
8
9
|
import { runFrameworkRules } from "../rules/frameworkRules.js";
|
|
9
10
|
import { runWebApiRules } from "../rules/webApiRules.js";
|
|
10
11
|
import { runTypedRequestBodyRules } from "../rules/typedRequestBodyRules.js";
|
|
12
|
+
import { runPythonSdkRules } from "../rules/pythonSdkRules.js";
|
|
13
|
+
import { runPythonFrameworkRules } from "../rules/pythonFrameworkRules.js";
|
|
11
14
|
import { findZodCandidates } from "../rules/zodPreFilter.js";
|
|
12
15
|
import { buildTsProject } from "./tsProject.js";
|
|
13
16
|
import { runHtmlInputRules, extractInlineScripts } from "../rules/htmlRules.js";
|
|
@@ -34,9 +37,11 @@ function makeLazyTsProject(files) {
|
|
|
34
37
|
return handle;
|
|
35
38
|
};
|
|
36
39
|
}
|
|
37
|
-
function runCodeRules(filePath, source, root, getTsProject, zodCandidates) {
|
|
40
|
+
function runCodeRules(filePath, source, root, getTsProject, zodCandidates, oversizedZodFiles) {
|
|
38
41
|
const ast = parseSource(source, filePath);
|
|
39
|
-
|
|
42
|
+
const zodResult = findZodCandidates(source, filePath, root);
|
|
43
|
+
zodCandidates.push(...zodResult.candidates);
|
|
44
|
+
oversizedZodFiles.push(...zodResult.oversizedFiles);
|
|
40
45
|
if (!ast)
|
|
41
46
|
return [];
|
|
42
47
|
return [
|
|
@@ -46,6 +51,12 @@ function runCodeRules(filePath, source, root, getTsProject, zodCandidates) {
|
|
|
46
51
|
...runTypedRequestBodyRules(ast, filePath, source, root, getTsProject),
|
|
47
52
|
];
|
|
48
53
|
}
|
|
54
|
+
async function runPythonCodeRules(filePath, source, root) {
|
|
55
|
+
const tree = await parsePythonSource(source);
|
|
56
|
+
if (!tree)
|
|
57
|
+
return [];
|
|
58
|
+
return [...runPythonSdkRules(tree, filePath, source, root), ...runPythonFrameworkRules(tree, filePath, source, root)];
|
|
59
|
+
}
|
|
49
60
|
/**
|
|
50
61
|
* Pure, local, offline — no network access anywhere in this function or anything it calls. Zod
|
|
51
62
|
* schemas and confidence scoring can no longer be fully resolved here (see pendingZodCandidates and
|
|
@@ -59,10 +70,12 @@ export async function scan(root) {
|
|
|
59
70
|
const [projects, unsupportedStackWarning] = await Promise.all([discoverProjects(root), detectUnsupportedStack(root)]);
|
|
60
71
|
const findings = [];
|
|
61
72
|
const pendingZodCandidates = [];
|
|
73
|
+
const oversizedZodFiles = [];
|
|
62
74
|
let filesScanned = 0;
|
|
63
75
|
const projectRootList = [...new Set(projects.map((p) => p.root))];
|
|
64
76
|
const fileSets = await Promise.all(projectRootList.map((projectRoot) => discoverFiles(projectRoot)));
|
|
65
77
|
const allCodeFiles = new Set();
|
|
78
|
+
const allPythonFiles = new Set();
|
|
66
79
|
const allHtmlFiles = new Set();
|
|
67
80
|
const allSchemaFiles = new Set();
|
|
68
81
|
const rootForFile = new Map();
|
|
@@ -74,6 +87,7 @@ export async function scan(root) {
|
|
|
74
87
|
allCodeFiles.add(f);
|
|
75
88
|
rootForFile.set(f, projectRoot);
|
|
76
89
|
});
|
|
90
|
+
set.pythonFiles.forEach((f) => allPythonFiles.add(f));
|
|
77
91
|
set.htmlFiles.forEach((f) => allHtmlFiles.add(f));
|
|
78
92
|
set.schemaFiles.forEach((f) => allSchemaFiles.add(f));
|
|
79
93
|
});
|
|
@@ -90,7 +104,18 @@ export async function scan(root) {
|
|
|
90
104
|
continue;
|
|
91
105
|
}
|
|
92
106
|
const getTsProject = tsProjectByRoot.get(rootForFile.get(filePath) ?? "") ?? noTsProject;
|
|
93
|
-
findings.push(...runCodeRules(filePath, source, root, getTsProject, pendingZodCandidates));
|
|
107
|
+
findings.push(...runCodeRules(filePath, source, root, getTsProject, pendingZodCandidates, oversizedZodFiles));
|
|
108
|
+
}
|
|
109
|
+
for (const filePath of allPythonFiles) {
|
|
110
|
+
filesScanned += 1;
|
|
111
|
+
let source;
|
|
112
|
+
try {
|
|
113
|
+
source = readFileSync(filePath, "utf8");
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
findings.push(...(await runPythonCodeRules(filePath, source, root)));
|
|
94
119
|
}
|
|
95
120
|
for (const filePath of allHtmlFiles) {
|
|
96
121
|
filesScanned += 1;
|
|
@@ -103,7 +128,7 @@ export async function scan(root) {
|
|
|
103
128
|
}
|
|
104
129
|
findings.push(...runHtmlInputRules(html, filePath, root));
|
|
105
130
|
for (const script of extractInlineScripts(html)) {
|
|
106
|
-
findings.push(...runCodeRules(filePath, script.code, root, noTsProject, pendingZodCandidates));
|
|
131
|
+
findings.push(...runCodeRules(filePath, script.code, root, noTsProject, pendingZodCandidates, oversizedZodFiles));
|
|
107
132
|
}
|
|
108
133
|
}
|
|
109
134
|
for (const filePath of allSchemaFiles) {
|
|
@@ -126,5 +151,17 @@ export async function scan(root) {
|
|
|
126
151
|
}
|
|
127
152
|
}
|
|
128
153
|
const ambiguousCount = findings.filter((f) => f.requiresReview).length;
|
|
129
|
-
|
|
154
|
+
const maxZodFragmentBytes = pendingZodCandidates.length
|
|
155
|
+
? Math.max(...pendingZodCandidates.map((c) => Buffer.byteLength(c.codeFragment, "utf8")))
|
|
156
|
+
: 0;
|
|
157
|
+
return {
|
|
158
|
+
projects,
|
|
159
|
+
findings,
|
|
160
|
+
ambiguousCount,
|
|
161
|
+
filesScanned,
|
|
162
|
+
unsupportedStackWarning,
|
|
163
|
+
pendingZodCandidates,
|
|
164
|
+
oversizedZodFiles,
|
|
165
|
+
maxZodFragmentBytes,
|
|
166
|
+
};
|
|
130
167
|
}
|
package/dist/engine/types.d.ts
CHANGED
|
@@ -68,4 +68,13 @@ export interface ScanResult {
|
|
|
68
68
|
codeFragment: string;
|
|
69
69
|
fragmentLineToSourceLine: number[];
|
|
70
70
|
}>;
|
|
71
|
+
/** Relative paths of files whose extracted Zod fragment was too large to send safely and was
|
|
72
|
+
* dropped before ever reaching pendingZodCandidates -- surfaced in resolveProtectedLogic()'s
|
|
73
|
+
* warning so a schema going unchecked is never silent, just like a failed network call is. */
|
|
74
|
+
oversizedZodFiles: string[];
|
|
75
|
+
/** The largest pendingZodCandidates fragment's byte size this scan, or 0 if none. Logged by the
|
|
76
|
+
* caller so real-world fragment sizes keep being observed against MAX_FRAGMENT_BYTES
|
|
77
|
+
* (zodPreFilter.ts) and the batching cap (protectedLogicClient.ts) -- both are provisional,
|
|
78
|
+
* unvalidated against the Worker's real limit until its plan upgrade is confirmed. */
|
|
79
|
+
maxZodFragmentBytes: number;
|
|
71
80
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2016 Max Brunsfeld
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
Binary file
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CONFIG_FILENAME } from "../config/config.js";
|
|
1
2
|
import { AnthropicProvider } from "./anthropicProvider.js";
|
|
2
3
|
export function resolveProviderForTier(config) {
|
|
3
4
|
const tier = config.tier ?? "free";
|
|
@@ -9,7 +10,7 @@ export function resolveProviderForTier(config) {
|
|
|
9
10
|
if (!envVar) {
|
|
10
11
|
return {
|
|
11
12
|
provider: null,
|
|
12
|
-
error:
|
|
13
|
+
error: `Tier is "byok" but no API key environment variable is configured in ${CONFIG_FILENAME}. Run "srclawyer init" to set one.`,
|
|
13
14
|
};
|
|
14
15
|
}
|
|
15
16
|
const apiKey = process.env[envVar];
|
|
@@ -27,5 +28,5 @@ export function resolveProviderForTier(config) {
|
|
|
27
28
|
error: 'Tier is "managed" but SrcLawyer\'s managed backend is not available yet. Use "byok" with your own Anthropic API key, or "free".',
|
|
28
29
|
};
|
|
29
30
|
}
|
|
30
|
-
return { provider: null, error: `Unknown tier "${tier}" in
|
|
31
|
+
return { provider: null, error: `Unknown tier "${tier}" in ${CONFIG_FILENAME}.` };
|
|
31
32
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { ScanResult } from "../engine/types.js";
|
|
2
|
-
import type
|
|
3
|
-
export declare function generatePolicyMarkdown(result: ScanResult, config: SrcLawyerConfig | null, llmError?: string | null): string;
|
|
2
|
+
import { type SrcLawyerConfig } from "../config/config.js";
|
|
3
|
+
export declare function generatePolicyMarkdown(result: ScanResult, config: SrcLawyerConfig | null, llmError?: string | null, protectedLogicWarning?: string | null): string;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CONFIG_FILENAME } from "../config/config.js";
|
|
1
2
|
import { DATA_CATEGORY_LABELS } from "./dataCategoryLabels.js";
|
|
2
3
|
import { MANDATORY_DISCLAIMER } from "./disclaimer.js";
|
|
3
4
|
function locationRef(finding) {
|
|
@@ -88,7 +89,7 @@ function renderContextSection(config) {
|
|
|
88
89
|
return [
|
|
89
90
|
"## Scope",
|
|
90
91
|
"",
|
|
91
|
-
|
|
92
|
+
`No \`${CONFIG_FILENAME}\` configuration was found, so this policy was generated without jurisdiction, target-market, or industry context. Run \`srclawyer init\` and re-run \`srclawyer scan\` to include that context.`,
|
|
92
93
|
].join("\n");
|
|
93
94
|
}
|
|
94
95
|
const lines = ["## Scope", ""];
|
|
@@ -101,6 +102,12 @@ function renderContextSection(config) {
|
|
|
101
102
|
function renderUnsupportedStackWarning(warning) {
|
|
102
103
|
return ["> **WARNING: INCOMPLETE SCAN — DO NOT PUBLISH WITHOUT MANUAL REVIEW**", ">", `> ${warning}`].join("\n");
|
|
103
104
|
}
|
|
105
|
+
// Mirrors renderUnsupportedStackWarning's own reasoning exactly: this policy FILE gets committed and
|
|
106
|
+
// shared on its own, without the CLI output that already shows this warning -- a reader of the file
|
|
107
|
+
// alone has no other way to know confidence grading or Zod-schema detection didn't run for this scan.
|
|
108
|
+
function renderProtectedLogicWarning(warning) {
|
|
109
|
+
return ["> **WARNING: PROTECTED-LOGIC SERVICE UNAVAILABLE — POLICY MAY BE INCOMPLETE**", ">", `> ${warning}`].join("\n");
|
|
110
|
+
}
|
|
104
111
|
function outcomeOf(finding) {
|
|
105
112
|
if (!finding.requiresReview)
|
|
106
113
|
return "confident";
|
|
@@ -110,7 +117,7 @@ function outcomeOf(finding) {
|
|
|
110
117
|
return "declined";
|
|
111
118
|
return "unresolved";
|
|
112
119
|
}
|
|
113
|
-
export function generatePolicyMarkdown(result, config, llmError) {
|
|
120
|
+
export function generatePolicyMarkdown(result, config, llmError, protectedLogicWarning) {
|
|
114
121
|
const confidentFindings = result.findings.filter((f) => {
|
|
115
122
|
const outcome = outcomeOf(f);
|
|
116
123
|
return outcome === "confident" || outcome === "resolved";
|
|
@@ -121,6 +128,9 @@ export function generatePolicyMarkdown(result, config, llmError) {
|
|
|
121
128
|
if (result.unsupportedStackWarning) {
|
|
122
129
|
parts.push(renderUnsupportedStackWarning(result.unsupportedStackWarning), "");
|
|
123
130
|
}
|
|
131
|
+
if (protectedLogicWarning) {
|
|
132
|
+
parts.push(renderProtectedLogicWarning(protectedLogicWarning), "");
|
|
133
|
+
}
|
|
124
134
|
if (llmError) {
|
|
125
135
|
parts.push(renderLlmErrorWarning(llmError), "");
|
|
126
136
|
}
|
|
@@ -129,6 +139,9 @@ export function generatePolicyMarkdown(result, config, llmError) {
|
|
|
129
139
|
if (categorySections.length > 0) {
|
|
130
140
|
parts.push(categorySections.join("\n\n"));
|
|
131
141
|
}
|
|
142
|
+
else if (result.unsupportedStackWarning) {
|
|
143
|
+
parts.push("**Not analyzed:** this codebase's primary language/framework isn't supported yet (see the warning above). This is not a clean-scan result — nothing below should be read as \"no data collection found.\"");
|
|
144
|
+
}
|
|
132
145
|
else {
|
|
133
146
|
parts.push("No data-collection patterns were confidently identified in this codebase.");
|
|
134
147
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare const PRIVACY_POLICY_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
2
|
+
export declare const TERMS_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
3
|
+
export declare const NETWORK_CALLS_NOTICE: string;
|
|
4
|
+
export declare const SCAN_DISCLAIMER_REMINDER = "Reminder: this is automated analysis, not legal advice \u2014 review the generated policy before publishing it.";
|
|
5
|
+
export declare function byokNetworkNotice(apiKeyEnvVar: string | null): string;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// No dedicated privacy-policy/terms site exists yet -- these point at the real, resolvable README
|
|
2
|
+
// "Legal" section that covers the same ground today (not a placeholder or a dead `.invalid` link), and
|
|
3
|
+
// should be repointed to a real site once one exists. Kept as named constants precisely so there's one
|
|
4
|
+
// place to update when that happens -- a one-line change, not a hunt through every call site.
|
|
5
|
+
//
|
|
6
|
+
// Deliberately NOT labeled "Privacy policy:" / "Terms:" below: that would claim those documents exist
|
|
7
|
+
// at this URL, when the Legal section itself says plainly that they don't yet -- this points at a
|
|
8
|
+
// notice and disclaimer, not a substitute for either.
|
|
9
|
+
export const PRIVACY_POLICY_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
10
|
+
export const TERMS_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
11
|
+
// Verbatim from README.md's own "Network calls" section (minus markdown backticks, since this prints
|
|
12
|
+
// to a terminal) -- one wording for this claim, not a second paraphrase that can drift from the first.
|
|
13
|
+
export const NETWORK_CALLS_NOTICE = [
|
|
14
|
+
"Heads up: SrcLawyer makes network calls.",
|
|
15
|
+
"",
|
|
16
|
+
"Detection itself is fully local and offline — no code leaves your machine for pattern matching.",
|
|
17
|
+
"Once detection completes, scan makes one batched network call to grade the confidence of what it",
|
|
18
|
+
"found (field names and derived yes/no signals only, never source code), and, for scans that use Zod",
|
|
19
|
+
"schemas as request validators, one additional call sending a small redacted code fragment. This runs",
|
|
20
|
+
"on every scan regardless of tier.",
|
|
21
|
+
"",
|
|
22
|
+
`Legal notice and disclaimer: ${PRIVACY_POLICY_URL}`,
|
|
23
|
+
"(A dedicated privacy policy and terms of service are coming; this points to what exists today.)",
|
|
24
|
+
].join("\n");
|
|
25
|
+
export const SCAN_DISCLAIMER_REMINDER = "Reminder: this is automated analysis, not legal advice — review the generated policy before publishing it.";
|
|
26
|
+
export function byokNetworkNotice(apiKeyEnvVar) {
|
|
27
|
+
const keySource = apiKeyEnvVar ? `env var ${apiKeyEnvVar}` : "your configured key";
|
|
28
|
+
return `BYOK is configured: redacted code snippets for ambiguous findings will be sent to Anthropic under your own API key (${keySource}), for AI-assisted classification only.`;
|
|
29
|
+
}
|
|
@@ -38,7 +38,7 @@ export function formatText(result) {
|
|
|
38
38
|
}
|
|
39
39
|
else if (result.ambiguousCount > 0) {
|
|
40
40
|
lines.push("");
|
|
41
|
-
lines.push(`${result.ambiguousCount} item(s) require deeper analysis before they can be included in a policy. Run "srclawyer scan" with
|
|
41
|
+
lines.push(`${result.ambiguousCount} item(s) require deeper analysis before they can be included in a policy. Run "srclawyer scan" with the "byok" tier configured (bring your own Anthropic key) to attempt AI-assisted resolution, or resolve manually. A "managed" tier is planned but not yet available.`);
|
|
42
42
|
}
|
|
43
43
|
return lines.join("\n");
|
|
44
44
|
}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
const LEXICON = [
|
|
2
2
|
{ pattern: /email/i, category: "email" },
|
|
3
|
-
// Checked before the generic "name" pattern below: bank_name/account_holder_name end in
|
|
4
|
-
// "name" and would otherwise be classified as just "name", losing the payment-specific signal.
|
|
5
3
|
{ pattern: /bank[ _-]?name|routing[ _-]?number|account[ _-]?holder|iban|swift[ _-]?code/i, category: "payment_info" },
|
|
6
|
-
|
|
4
|
+
// Anchored to the whole field name, not a bare "...name$" suffix match: the old pattern matched
|
|
5
|
+
// ANY field ending in "name" -- hostname, filename, directoryName, className, teamName, etc. -- none
|
|
6
|
+
// of which are a person's name. This matches only "name" itself and the known person-name
|
|
7
|
+
// decompositions (first/last/middle/maiden/nick/display/legal name, username), each allowing an
|
|
8
|
+
// optional separator (first_name, firstName, first-name). A compound this doesn't recognize falls
|
|
9
|
+
// through with no category match, landing in the same "could not confidently classify — needs
|
|
10
|
+
// review" path as any other unclassified field -- not the safe list, not a new discard path, just
|
|
11
|
+
// the existing honest-ambiguity treatment.
|
|
12
|
+
{ pattern: /^(full|first|middle|last|sur|maiden|nick|display|legal)[\s_-]*name$|^name$|^user[\s_-]*name$/i, category: "name" },
|
|
7
13
|
{ pattern: /phone|mobile|telephone/i, category: "phone" },
|
|
8
14
|
{ pattern: /address|street|city|zip ?code|postal ?code/i, category: "physical_address" },
|
|
9
15
|
{ pattern: /card ?number|creditcard|cvv|cvc|cardholder/i, category: "payment_info" },
|
|
@@ -18,6 +18,54 @@ function fieldsFromObjectPattern(pattern) {
|
|
|
18
18
|
}
|
|
19
19
|
return names;
|
|
20
20
|
}
|
|
21
|
+
function isRequestBodyMemberExpression(node) {
|
|
22
|
+
return (node.type === "MemberExpression" &&
|
|
23
|
+
!node.computed &&
|
|
24
|
+
node.object.type === "Identifier" &&
|
|
25
|
+
REQUEST_OBJECT_NAMES.test(node.object.name) &&
|
|
26
|
+
node.property.type === "Identifier" &&
|
|
27
|
+
REQUEST_BODY_PROPS.test(node.property.name));
|
|
28
|
+
}
|
|
29
|
+
function isAwaitedRequestJsonCall(node) {
|
|
30
|
+
return (node.type === "AwaitExpression" &&
|
|
31
|
+
node.argument.type === "CallExpression" &&
|
|
32
|
+
node.argument.callee.type === "MemberExpression" &&
|
|
33
|
+
node.argument.callee.object.type === "Identifier" &&
|
|
34
|
+
REQUEST_OBJECT_NAMES.test(node.argument.callee.object.name) &&
|
|
35
|
+
node.argument.callee.property.type === "Identifier" &&
|
|
36
|
+
node.argument.callee.property.name === "json");
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Same-function alias tracking: `const body = req.body` (or `req.query`/`req.params`, or
|
|
40
|
+
* `await req.json()`) followed by `body.email` / `body['email']` / `const { email } = body` in the
|
|
41
|
+
* same function body. Deliberately one mechanical hop, scoped to the nearest enclosing function (or
|
|
42
|
+
* module/program scope for top-level code) via Babel's own `getFunctionParent()` -- not general
|
|
43
|
+
* dataflow: no cross-function tracking, no chained reassignment, no closures capturing an outer
|
|
44
|
+
* function's alias. Mirrors pythonFrameworkRules.ts's same-function alias tracking exactly, for the
|
|
45
|
+
* same reason: `const body = req.body; body.email` was structurally invisible before this, just like
|
|
46
|
+
* `data = request.get_json(); data['email']` was on the Python side.
|
|
47
|
+
*/
|
|
48
|
+
function collectRequestAliases(ast) {
|
|
49
|
+
const aliasesByScope = new Map();
|
|
50
|
+
traverse(ast, {
|
|
51
|
+
VariableDeclarator(path) {
|
|
52
|
+
const id = path.node.id;
|
|
53
|
+
const init = path.node.init;
|
|
54
|
+
if (!init || id.type !== "Identifier")
|
|
55
|
+
return;
|
|
56
|
+
if (!isRequestBodyMemberExpression(init) && !isAwaitedRequestJsonCall(init))
|
|
57
|
+
return;
|
|
58
|
+
const scope = path.getFunctionParent()?.node ?? ast.program;
|
|
59
|
+
const aliases = aliasesByScope.get(scope) ?? new Set();
|
|
60
|
+
aliases.add(id.name);
|
|
61
|
+
aliasesByScope.set(scope, aliases);
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
return aliasesByScope;
|
|
65
|
+
}
|
|
66
|
+
function enclosingScope(path, ast) {
|
|
67
|
+
return path.getFunctionParent()?.node ?? ast.program;
|
|
68
|
+
}
|
|
21
69
|
export function runFrameworkRules(ast, filePath, source, root) {
|
|
22
70
|
const findings = [];
|
|
23
71
|
const lines = source.split("\n");
|
|
@@ -53,24 +101,38 @@ export function runFrameworkRules(ast, filePath, source, root) {
|
|
|
53
101
|
confidenceFactors: { lexiconCategory, ambiguousName, runtimeVerified: true, siblingConfidentCount },
|
|
54
102
|
});
|
|
55
103
|
}
|
|
104
|
+
const aliasesByScope = collectRequestAliases(ast);
|
|
105
|
+
const isAliasReference = (path, name) => !!aliasesByScope.get(enclosingScope(path, ast))?.has(name);
|
|
56
106
|
traverse(ast, {
|
|
57
107
|
MemberExpression(path) {
|
|
58
|
-
const { object, property } = path.node;
|
|
59
|
-
if (object.type !== "Identifier"
|
|
108
|
+
const { object, property, computed } = path.node;
|
|
109
|
+
if (object.type !== "Identifier")
|
|
60
110
|
return;
|
|
61
|
-
if (property.type
|
|
111
|
+
if (REQUEST_OBJECT_NAMES.test(object.name) && !computed && property.type === "Identifier" && REQUEST_BODY_PROPS.test(property.name)) {
|
|
112
|
+
const parent = path.parent;
|
|
113
|
+
if (parent.type === "MemberExpression" && parent.property.type === "Identifier") {
|
|
114
|
+
emit(parent.property.name, parent.loc?.start.line ?? path.node.loc?.start.line ?? 1, parent);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (parent.type === "VariableDeclarator" && parent.id.type === "ObjectPattern") {
|
|
118
|
+
const line = parent.loc?.start.line ?? 1;
|
|
119
|
+
const siblingFieldNames = fieldsFromObjectPattern(parent.id);
|
|
120
|
+
for (const fieldName of siblingFieldNames) {
|
|
121
|
+
emit(fieldName, line, parent, siblingFieldNames);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
62
124
|
return;
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
125
|
+
}
|
|
126
|
+
// Alias reference: `body.email` or `body['email']`, where `body` was assigned from
|
|
127
|
+
// req.body/.query/.params or await req.json() earlier in the same function.
|
|
128
|
+
if (!isAliasReference(path, object.name))
|
|
66
129
|
return;
|
|
130
|
+
const line = path.node.loc?.start.line ?? 1;
|
|
131
|
+
if (!computed && property.type === "Identifier") {
|
|
132
|
+
emit(property.name, line, path.node);
|
|
67
133
|
}
|
|
68
|
-
if (
|
|
69
|
-
|
|
70
|
-
const siblingFieldNames = fieldsFromObjectPattern(parent.id);
|
|
71
|
-
for (const fieldName of siblingFieldNames) {
|
|
72
|
-
emit(fieldName, line, parent, siblingFieldNames);
|
|
73
|
-
}
|
|
134
|
+
else if (computed && property.type === "StringLiteral") {
|
|
135
|
+
emit(property.value, line, path.node);
|
|
74
136
|
}
|
|
75
137
|
},
|
|
76
138
|
AwaitExpression(path) {
|
|
@@ -93,6 +155,21 @@ export function runFrameworkRules(ast, filePath, source, root) {
|
|
|
93
155
|
}
|
|
94
156
|
}
|
|
95
157
|
},
|
|
158
|
+
VariableDeclarator(path) {
|
|
159
|
+
// Alias destructure: `const { email } = body`, where `body` was assigned from
|
|
160
|
+
// req.body/.query/.params or await req.json() earlier in the same function.
|
|
161
|
+
const init = path.node.init;
|
|
162
|
+
const id = path.node.id;
|
|
163
|
+
if (!init || init.type !== "Identifier" || id.type !== "ObjectPattern")
|
|
164
|
+
return;
|
|
165
|
+
if (!isAliasReference(path, init.name))
|
|
166
|
+
return;
|
|
167
|
+
const line = path.node.loc?.start.line ?? 1;
|
|
168
|
+
const siblingFieldNames = fieldsFromObjectPattern(id);
|
|
169
|
+
for (const fieldName of siblingFieldNames) {
|
|
170
|
+
emit(fieldName, line, path.node, siblingFieldNames);
|
|
171
|
+
}
|
|
172
|
+
},
|
|
96
173
|
});
|
|
97
174
|
return findings;
|
|
98
175
|
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { relative } from "node:path";
|
|
2
|
+
import { classifyFieldName, isAmbiguousFieldName } from "./dataCategoryLexicon.js";
|
|
3
|
+
import { isSafeFieldName } from "./safeFieldNames.js";
|
|
4
|
+
import { redactSecrets } from "../engine/secretRedaction.js";
|
|
5
|
+
import { countConfidentSiblings } from "./confidenceFactors.js";
|
|
6
|
+
/**
|
|
7
|
+
* Flask's incoming-request accessors: request.form / request.json / request.args. Anchored to the
|
|
8
|
+
* literal name "request" rather than a configurable-name regex (unlike Express's req/request, which
|
|
9
|
+
* is a per-handler parameter that can be named anything) -- Flask's `request` is always the direct,
|
|
10
|
+
* module-level import from `flask`, used by that exact name in effectively all real code.
|
|
11
|
+
*/
|
|
12
|
+
const REQUEST_ATTR = /^request\.(form|json|args)$/;
|
|
13
|
+
const REQUEST_GET_CALL = /^request\.(form|json|args)\.get$/;
|
|
14
|
+
const REQUEST_GET_JSON_CALL = /^request\.get_json$/;
|
|
15
|
+
function stringLiteralContent(node) {
|
|
16
|
+
if (!node || node.type !== "string")
|
|
17
|
+
return null;
|
|
18
|
+
return node.descendantsOfType("string_content")[0]?.text ?? null;
|
|
19
|
+
}
|
|
20
|
+
/** True for the RHS of `x = <this>`: request.form / .json / .args, or a request.get_json() call. */
|
|
21
|
+
function isRequestDataSource(node) {
|
|
22
|
+
if (node.type === "attribute")
|
|
23
|
+
return REQUEST_ATTR.test(node.text);
|
|
24
|
+
if (node.type === "call") {
|
|
25
|
+
const fn = node.childForFieldName("function");
|
|
26
|
+
return !!fn && REQUEST_GET_JSON_CALL.test(fn.text);
|
|
27
|
+
}
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The nearest enclosing function_definition's node id, or -1 for module-level code. Scoped by `.id`
|
|
32
|
+
* rather than the Node object itself -- web-tree-sitter doesn't guarantee the same underlying syntax
|
|
33
|
+
* node comes back as the same JS object identity across separate traversal calls (same `.id`, new
|
|
34
|
+
* wrapper), so using Node references as Map keys silently never matches.
|
|
35
|
+
*/
|
|
36
|
+
function enclosingScopeId(node) {
|
|
37
|
+
let current = node.parent;
|
|
38
|
+
while (current) {
|
|
39
|
+
if (current.type === "function_definition")
|
|
40
|
+
return current.id;
|
|
41
|
+
current = current.parent;
|
|
42
|
+
}
|
|
43
|
+
return -1;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Same-function alias tracking: `data = request.get_json()` (or `= request.form` / `.json` / `.args`)
|
|
47
|
+
* followed by `data['x']` / `data.get('x')` in the same function body. Deliberately one mechanical hop,
|
|
48
|
+
* scoped to the nearest enclosing function (or module level), not general dataflow -- no cross-function
|
|
49
|
+
* tracking, no chained reassignment, no branch/conditional awareness, no closures capturing an outer
|
|
50
|
+
* function's alias. This is exactly the gap the microblog golden-corpus entry documented: real signup
|
|
51
|
+
* PII read via request.get_json() into a local variable rather than a direct request.json[...] subscript.
|
|
52
|
+
*/
|
|
53
|
+
function collectRequestAliases(tree) {
|
|
54
|
+
const aliasesByScope = new Map();
|
|
55
|
+
for (const node of tree.rootNode.descendantsOfType("assignment")) {
|
|
56
|
+
const left = node.childForFieldName("left");
|
|
57
|
+
const right = node.childForFieldName("right");
|
|
58
|
+
if (!left || left.type !== "identifier" || !right || !isRequestDataSource(right))
|
|
59
|
+
continue;
|
|
60
|
+
const scope = enclosingScopeId(node);
|
|
61
|
+
const aliases = aliasesByScope.get(scope) ?? new Set();
|
|
62
|
+
aliases.add(left.text);
|
|
63
|
+
aliasesByScope.set(scope, aliases);
|
|
64
|
+
}
|
|
65
|
+
return aliasesByScope;
|
|
66
|
+
}
|
|
67
|
+
function collectFieldAccesses(tree) {
|
|
68
|
+
const accesses = [];
|
|
69
|
+
const aliasesByScope = collectRequestAliases(tree);
|
|
70
|
+
const isAliasReference = (node) => !!node && node.type === "identifier" && !!aliasesByScope.get(enclosingScopeId(node))?.has(node.text);
|
|
71
|
+
for (const node of tree.rootNode.descendantsOfType("subscript")) {
|
|
72
|
+
const value = node.childForFieldName("value");
|
|
73
|
+
if (!value)
|
|
74
|
+
continue;
|
|
75
|
+
const isDirect = REQUEST_ATTR.test(value.text) || (value.type === "call" && isRequestDataSource(value));
|
|
76
|
+
if (!isDirect && !isAliasReference(value))
|
|
77
|
+
continue;
|
|
78
|
+
const key = stringLiteralContent(node.childForFieldName("subscript"));
|
|
79
|
+
if (key)
|
|
80
|
+
accesses.push({ fieldName: key, line: node.startPosition.row + 1 });
|
|
81
|
+
}
|
|
82
|
+
for (const node of tree.rootNode.descendantsOfType("call")) {
|
|
83
|
+
const fn = node.childForFieldName("function");
|
|
84
|
+
if (!fn)
|
|
85
|
+
continue;
|
|
86
|
+
const isDirectGet = REQUEST_GET_CALL.test(fn.text);
|
|
87
|
+
const isAliasGet = fn.type === "attribute" &&
|
|
88
|
+
fn.childForFieldName("attribute")?.text === "get" &&
|
|
89
|
+
isAliasReference(fn.childForFieldName("object"));
|
|
90
|
+
if (!isDirectGet && !isAliasGet)
|
|
91
|
+
continue;
|
|
92
|
+
const args = node.childForFieldName("arguments")?.namedChildren ?? [];
|
|
93
|
+
const key = stringLiteralContent(args[0] ?? null);
|
|
94
|
+
if (key)
|
|
95
|
+
accesses.push({ fieldName: key, line: node.startPosition.row + 1 });
|
|
96
|
+
}
|
|
97
|
+
return accesses;
|
|
98
|
+
}
|
|
99
|
+
export function runPythonFrameworkRules(tree, filePath, source, root) {
|
|
100
|
+
const lines = source.split("\n");
|
|
101
|
+
const relPath = relative(root, filePath);
|
|
102
|
+
const accesses = collectFieldAccesses(tree);
|
|
103
|
+
const allFieldNames = accesses.map((a) => a.fieldName);
|
|
104
|
+
const findings = [];
|
|
105
|
+
const emitted = new Set();
|
|
106
|
+
for (const { fieldName, line } of accesses) {
|
|
107
|
+
if (isSafeFieldName(fieldName))
|
|
108
|
+
continue;
|
|
109
|
+
const dedupeKey = `${fieldName}:${line}`;
|
|
110
|
+
if (emitted.has(dedupeKey))
|
|
111
|
+
continue;
|
|
112
|
+
emitted.add(dedupeKey);
|
|
113
|
+
const lexiconCategory = classifyFieldName(fieldName);
|
|
114
|
+
const ambiguousName = isAmbiguousFieldName(fieldName);
|
|
115
|
+
const siblingConfidentCount = countConfidentSiblings(allFieldNames, fieldName);
|
|
116
|
+
const evidenceLine = lines[line - 1] ?? "";
|
|
117
|
+
// Provisional confidence, patched by the batched protected-logic scoring call -- same pattern as
|
|
118
|
+
// frameworkRules.ts's JS equivalent; the scoring endpoint doesn't care which language a
|
|
119
|
+
// confidenceFactors-bearing finding came from.
|
|
120
|
+
findings.push({
|
|
121
|
+
id: `field:${relPath}:${line}:${fieldName}`,
|
|
122
|
+
dataCategories: lexiconCategory ? [lexiconCategory] : ["generic_pii"],
|
|
123
|
+
processor: null,
|
|
124
|
+
description: lexiconCategory
|
|
125
|
+
? `Request field "${fieldName}" collected, classified as ${lexiconCategory}.`
|
|
126
|
+
: `Request field "${fieldName}" collected; could not confidently classify — needs review.`,
|
|
127
|
+
confidence: "low",
|
|
128
|
+
source: "framework-rule",
|
|
129
|
+
location: { file: relPath, line, column: 0 },
|
|
130
|
+
evidence: redactSecrets(evidenceLine.trim()).slice(0, 200),
|
|
131
|
+
requiresReview: true,
|
|
132
|
+
confidenceFactors: { lexiconCategory, ambiguousName, runtimeVerified: true, siblingConfidentCount },
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
return findings;
|
|
136
|
+
}
|