srclawyer 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -6
- package/dist/cloud/protectedLogicClient.d.ts +3 -0
- package/dist/cloud/protectedLogicClient.js +62 -20
- package/dist/cloud/resolveProtectedLogic.d.ts +1 -1
- package/dist/cloud/resolveProtectedLogic.js +15 -1
- package/dist/commands/init.js +9 -1
- package/dist/commands/scan.js +14 -3
- package/dist/engine/discoverFiles.d.ts +1 -0
- package/dist/engine/discoverFiles.js +16 -2
- package/dist/engine/languageDetection.js +37 -7
- package/dist/engine/pythonAstUtils.d.ts +2 -0
- package/dist/engine/pythonAstUtils.js +44 -0
- package/dist/engine/scanEngine.js +42 -5
- package/dist/engine/types.d.ts +9 -0
- package/dist/grammars/LICENSE +21 -0
- package/dist/grammars/tree-sitter-python.wasm +0 -0
- package/dist/policy/generatePolicy.d.ts +1 -1
- package/dist/policy/generatePolicy.js +10 -1
- package/dist/policy/legalNotices.d.ts +5 -0
- package/dist/policy/legalNotices.js +29 -0
- package/dist/report/formatText.js +1 -1
- package/dist/rules/dataCategoryLexicon.js +9 -3
- package/dist/rules/frameworkRules.js +89 -12
- package/dist/rules/pythonFrameworkRules.d.ts +3 -0
- package/dist/rules/pythonFrameworkRules.js +136 -0
- package/dist/rules/pythonSdkRules.d.ts +3 -0
- package/dist/rules/pythonSdkRules.js +73 -0
- package/dist/rules/safeFieldNames.js +14 -0
- package/dist/rules/zodPreFilter.d.ts +21 -1
- package/dist/rules/zodPreFilter.js +104 -13
- package/package.json +3 -2
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2016 Max Brunsfeld
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
Binary file
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { ScanResult } from "../engine/types.js";
|
|
2
2
|
import { type SrcLawyerConfig } from "../config/config.js";
|
|
3
|
-
export declare function generatePolicyMarkdown(result: ScanResult, config: SrcLawyerConfig | null, llmError?: string | null): string;
|
|
3
|
+
export declare function generatePolicyMarkdown(result: ScanResult, config: SrcLawyerConfig | null, llmError?: string | null, protectedLogicWarning?: string | null): string;
|
|
@@ -102,6 +102,12 @@ function renderContextSection(config) {
|
|
|
102
102
|
function renderUnsupportedStackWarning(warning) {
|
|
103
103
|
return ["> **WARNING: INCOMPLETE SCAN — DO NOT PUBLISH WITHOUT MANUAL REVIEW**", ">", `> ${warning}`].join("\n");
|
|
104
104
|
}
|
|
105
|
+
// Mirrors renderUnsupportedStackWarning's own reasoning exactly: this policy FILE gets committed and
|
|
106
|
+
// shared on its own, without the CLI output that already shows this warning -- a reader of the file
|
|
107
|
+
// alone has no other way to know confidence grading or Zod-schema detection didn't run for this scan.
|
|
108
|
+
function renderProtectedLogicWarning(warning) {
|
|
109
|
+
return ["> **WARNING: PROTECTED-LOGIC SERVICE UNAVAILABLE — POLICY MAY BE INCOMPLETE**", ">", `> ${warning}`].join("\n");
|
|
110
|
+
}
|
|
105
111
|
function outcomeOf(finding) {
|
|
106
112
|
if (!finding.requiresReview)
|
|
107
113
|
return "confident";
|
|
@@ -111,7 +117,7 @@ function outcomeOf(finding) {
|
|
|
111
117
|
return "declined";
|
|
112
118
|
return "unresolved";
|
|
113
119
|
}
|
|
114
|
-
export function generatePolicyMarkdown(result, config, llmError) {
|
|
120
|
+
export function generatePolicyMarkdown(result, config, llmError, protectedLogicWarning) {
|
|
115
121
|
const confidentFindings = result.findings.filter((f) => {
|
|
116
122
|
const outcome = outcomeOf(f);
|
|
117
123
|
return outcome === "confident" || outcome === "resolved";
|
|
@@ -122,6 +128,9 @@ export function generatePolicyMarkdown(result, config, llmError) {
|
|
|
122
128
|
if (result.unsupportedStackWarning) {
|
|
123
129
|
parts.push(renderUnsupportedStackWarning(result.unsupportedStackWarning), "");
|
|
124
130
|
}
|
|
131
|
+
if (protectedLogicWarning) {
|
|
132
|
+
parts.push(renderProtectedLogicWarning(protectedLogicWarning), "");
|
|
133
|
+
}
|
|
125
134
|
if (llmError) {
|
|
126
135
|
parts.push(renderLlmErrorWarning(llmError), "");
|
|
127
136
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare const PRIVACY_POLICY_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
2
|
+
export declare const TERMS_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
3
|
+
export declare const NETWORK_CALLS_NOTICE: string;
|
|
4
|
+
export declare const SCAN_DISCLAIMER_REMINDER = "Reminder: this is automated analysis, not legal advice \u2014 review the generated policy before publishing it.";
|
|
5
|
+
export declare function byokNetworkNotice(apiKeyEnvVar: string | null): string;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// No dedicated privacy-policy/terms site exists yet -- these point at the real, resolvable README
|
|
2
|
+
// "Legal" section that covers the same ground today (not a placeholder or a dead `.invalid` link), and
|
|
3
|
+
// should be repointed to a real site once one exists. Kept as named constants precisely so there's one
|
|
4
|
+
// place to update when that happens -- a one-line change, not a hunt through every call site.
|
|
5
|
+
//
|
|
6
|
+
// Deliberately NOT labeled "Privacy policy:" / "Terms:" below: that would claim those documents exist
|
|
7
|
+
// at this URL, when the Legal section itself says plainly that they don't yet -- this points at a
|
|
8
|
+
// notice and disclaimer, not a substitute for either.
|
|
9
|
+
export const PRIVACY_POLICY_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
10
|
+
export const TERMS_URL = "https://github.com/SrcLawyer/srclawyer#legal";
|
|
11
|
+
// Verbatim from README.md's own "Network calls" section (minus markdown backticks, since this prints
|
|
12
|
+
// to a terminal) -- one wording for this claim, not a second paraphrase that can drift from the first.
|
|
13
|
+
export const NETWORK_CALLS_NOTICE = [
|
|
14
|
+
"Heads up: SrcLawyer makes network calls.",
|
|
15
|
+
"",
|
|
16
|
+
"Detection itself is fully local and offline — no code leaves your machine for pattern matching.",
|
|
17
|
+
"Once detection completes, scan makes one batched network call to grade the confidence of what it",
|
|
18
|
+
"found (field names and derived yes/no signals only, never source code), and, for scans that use Zod",
|
|
19
|
+
"schemas as request validators, one additional call sending a small redacted code fragment. This runs",
|
|
20
|
+
"on every scan regardless of tier.",
|
|
21
|
+
"",
|
|
22
|
+
`Legal notice and disclaimer: ${PRIVACY_POLICY_URL}`,
|
|
23
|
+
"(A dedicated privacy policy and terms of service are coming; this points to what exists today.)",
|
|
24
|
+
].join("\n");
|
|
25
|
+
export const SCAN_DISCLAIMER_REMINDER = "Reminder: this is automated analysis, not legal advice — review the generated policy before publishing it.";
|
|
26
|
+
export function byokNetworkNotice(apiKeyEnvVar) {
|
|
27
|
+
const keySource = apiKeyEnvVar ? `env var ${apiKeyEnvVar}` : "your configured key";
|
|
28
|
+
return `BYOK is configured: redacted code snippets for ambiguous findings will be sent to Anthropic under your own API key (${keySource}), for AI-assisted classification only.`;
|
|
29
|
+
}
|
|
@@ -38,7 +38,7 @@ export function formatText(result) {
|
|
|
38
38
|
}
|
|
39
39
|
else if (result.ambiguousCount > 0) {
|
|
40
40
|
lines.push("");
|
|
41
|
-
lines.push(`${result.ambiguousCount} item(s) require deeper analysis before they can be included in a policy. Run "srclawyer scan" with
|
|
41
|
+
lines.push(`${result.ambiguousCount} item(s) require deeper analysis before they can be included in a policy. Run "srclawyer scan" with the "byok" tier configured (bring your own Anthropic key) to attempt AI-assisted resolution, or resolve manually. A "managed" tier is planned but not yet available.`);
|
|
42
42
|
}
|
|
43
43
|
return lines.join("\n");
|
|
44
44
|
}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
const LEXICON = [
|
|
2
2
|
{ pattern: /email/i, category: "email" },
|
|
3
|
-
// Checked before the generic "name" pattern below: bank_name/account_holder_name end in
|
|
4
|
-
// "name" and would otherwise be classified as just "name", losing the payment-specific signal.
|
|
5
3
|
{ pattern: /bank[ _-]?name|routing[ _-]?number|account[ _-]?holder|iban|swift[ _-]?code/i, category: "payment_info" },
|
|
6
|
-
|
|
4
|
+
// Anchored to the whole field name, not a bare "...name$" suffix match: the old pattern matched
|
|
5
|
+
// ANY field ending in "name" -- hostname, filename, directoryName, className, teamName, etc. -- none
|
|
6
|
+
// of which are a person's name. This matches only "name" itself and the known person-name
|
|
7
|
+
// decompositions (first/last/middle/maiden/nick/display/legal name, username), each allowing an
|
|
8
|
+
// optional separator (first_name, firstName, first-name). A compound this doesn't recognize falls
|
|
9
|
+
// through with no category match, landing in the same "could not confidently classify — needs
|
|
10
|
+
// review" path as any other unclassified field -- not the safe list, not a new discard path, just
|
|
11
|
+
// the existing honest-ambiguity treatment.
|
|
12
|
+
{ pattern: /^(full|first|middle|last|sur|maiden|nick|display|legal)[\s_-]*name$|^name$|^user[\s_-]*name$/i, category: "name" },
|
|
7
13
|
{ pattern: /phone|mobile|telephone/i, category: "phone" },
|
|
8
14
|
{ pattern: /address|street|city|zip ?code|postal ?code/i, category: "physical_address" },
|
|
9
15
|
{ pattern: /card ?number|creditcard|cvv|cvc|cardholder/i, category: "payment_info" },
|
|
@@ -18,6 +18,54 @@ function fieldsFromObjectPattern(pattern) {
|
|
|
18
18
|
}
|
|
19
19
|
return names;
|
|
20
20
|
}
|
|
21
|
+
function isRequestBodyMemberExpression(node) {
|
|
22
|
+
return (node.type === "MemberExpression" &&
|
|
23
|
+
!node.computed &&
|
|
24
|
+
node.object.type === "Identifier" &&
|
|
25
|
+
REQUEST_OBJECT_NAMES.test(node.object.name) &&
|
|
26
|
+
node.property.type === "Identifier" &&
|
|
27
|
+
REQUEST_BODY_PROPS.test(node.property.name));
|
|
28
|
+
}
|
|
29
|
+
function isAwaitedRequestJsonCall(node) {
|
|
30
|
+
return (node.type === "AwaitExpression" &&
|
|
31
|
+
node.argument.type === "CallExpression" &&
|
|
32
|
+
node.argument.callee.type === "MemberExpression" &&
|
|
33
|
+
node.argument.callee.object.type === "Identifier" &&
|
|
34
|
+
REQUEST_OBJECT_NAMES.test(node.argument.callee.object.name) &&
|
|
35
|
+
node.argument.callee.property.type === "Identifier" &&
|
|
36
|
+
node.argument.callee.property.name === "json");
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Same-function alias tracking: `const body = req.body` (or `req.query`/`req.params`, or
|
|
40
|
+
* `await req.json()`) followed by `body.email` / `body['email']` / `const { email } = body` in the
|
|
41
|
+
* same function body. Deliberately one mechanical hop, scoped to the nearest enclosing function (or
|
|
42
|
+
* module/program scope for top-level code) via Babel's own `getFunctionParent()` -- not general
|
|
43
|
+
* dataflow: no cross-function tracking, no chained reassignment, no closures capturing an outer
|
|
44
|
+
* function's alias. Mirrors pythonFrameworkRules.ts's same-function alias tracking exactly, for the
|
|
45
|
+
* same reason: `const body = req.body; body.email` was structurally invisible before this, just like
|
|
46
|
+
* `data = request.get_json(); data['email']` was on the Python side.
|
|
47
|
+
*/
|
|
48
|
+
function collectRequestAliases(ast) {
|
|
49
|
+
const aliasesByScope = new Map();
|
|
50
|
+
traverse(ast, {
|
|
51
|
+
VariableDeclarator(path) {
|
|
52
|
+
const id = path.node.id;
|
|
53
|
+
const init = path.node.init;
|
|
54
|
+
if (!init || id.type !== "Identifier")
|
|
55
|
+
return;
|
|
56
|
+
if (!isRequestBodyMemberExpression(init) && !isAwaitedRequestJsonCall(init))
|
|
57
|
+
return;
|
|
58
|
+
const scope = path.getFunctionParent()?.node ?? ast.program;
|
|
59
|
+
const aliases = aliasesByScope.get(scope) ?? new Set();
|
|
60
|
+
aliases.add(id.name);
|
|
61
|
+
aliasesByScope.set(scope, aliases);
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
return aliasesByScope;
|
|
65
|
+
}
|
|
66
|
+
function enclosingScope(path, ast) {
|
|
67
|
+
return path.getFunctionParent()?.node ?? ast.program;
|
|
68
|
+
}
|
|
21
69
|
export function runFrameworkRules(ast, filePath, source, root) {
|
|
22
70
|
const findings = [];
|
|
23
71
|
const lines = source.split("\n");
|
|
@@ -53,24 +101,38 @@ export function runFrameworkRules(ast, filePath, source, root) {
|
|
|
53
101
|
confidenceFactors: { lexiconCategory, ambiguousName, runtimeVerified: true, siblingConfidentCount },
|
|
54
102
|
});
|
|
55
103
|
}
|
|
104
|
+
const aliasesByScope = collectRequestAliases(ast);
|
|
105
|
+
const isAliasReference = (path, name) => !!aliasesByScope.get(enclosingScope(path, ast))?.has(name);
|
|
56
106
|
traverse(ast, {
|
|
57
107
|
MemberExpression(path) {
|
|
58
|
-
const { object, property } = path.node;
|
|
59
|
-
if (object.type !== "Identifier"
|
|
108
|
+
const { object, property, computed } = path.node;
|
|
109
|
+
if (object.type !== "Identifier")
|
|
60
110
|
return;
|
|
61
|
-
if (property.type
|
|
111
|
+
if (REQUEST_OBJECT_NAMES.test(object.name) && !computed && property.type === "Identifier" && REQUEST_BODY_PROPS.test(property.name)) {
|
|
112
|
+
const parent = path.parent;
|
|
113
|
+
if (parent.type === "MemberExpression" && parent.property.type === "Identifier") {
|
|
114
|
+
emit(parent.property.name, parent.loc?.start.line ?? path.node.loc?.start.line ?? 1, parent);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (parent.type === "VariableDeclarator" && parent.id.type === "ObjectPattern") {
|
|
118
|
+
const line = parent.loc?.start.line ?? 1;
|
|
119
|
+
const siblingFieldNames = fieldsFromObjectPattern(parent.id);
|
|
120
|
+
for (const fieldName of siblingFieldNames) {
|
|
121
|
+
emit(fieldName, line, parent, siblingFieldNames);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
62
124
|
return;
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
125
|
+
}
|
|
126
|
+
// Alias reference: `body.email` or `body['email']`, where `body` was assigned from
|
|
127
|
+
// req.body/.query/.params or await req.json() earlier in the same function.
|
|
128
|
+
if (!isAliasReference(path, object.name))
|
|
66
129
|
return;
|
|
130
|
+
const line = path.node.loc?.start.line ?? 1;
|
|
131
|
+
if (!computed && property.type === "Identifier") {
|
|
132
|
+
emit(property.name, line, path.node);
|
|
67
133
|
}
|
|
68
|
-
if (
|
|
69
|
-
|
|
70
|
-
const siblingFieldNames = fieldsFromObjectPattern(parent.id);
|
|
71
|
-
for (const fieldName of siblingFieldNames) {
|
|
72
|
-
emit(fieldName, line, parent, siblingFieldNames);
|
|
73
|
-
}
|
|
134
|
+
else if (computed && property.type === "StringLiteral") {
|
|
135
|
+
emit(property.value, line, path.node);
|
|
74
136
|
}
|
|
75
137
|
},
|
|
76
138
|
AwaitExpression(path) {
|
|
@@ -93,6 +155,21 @@ export function runFrameworkRules(ast, filePath, source, root) {
|
|
|
93
155
|
}
|
|
94
156
|
}
|
|
95
157
|
},
|
|
158
|
+
VariableDeclarator(path) {
|
|
159
|
+
// Alias destructure: `const { email } = body`, where `body` was assigned from
|
|
160
|
+
// req.body/.query/.params or await req.json() earlier in the same function.
|
|
161
|
+
const init = path.node.init;
|
|
162
|
+
const id = path.node.id;
|
|
163
|
+
if (!init || init.type !== "Identifier" || id.type !== "ObjectPattern")
|
|
164
|
+
return;
|
|
165
|
+
if (!isAliasReference(path, init.name))
|
|
166
|
+
return;
|
|
167
|
+
const line = path.node.loc?.start.line ?? 1;
|
|
168
|
+
const siblingFieldNames = fieldsFromObjectPattern(id);
|
|
169
|
+
for (const fieldName of siblingFieldNames) {
|
|
170
|
+
emit(fieldName, line, path.node, siblingFieldNames);
|
|
171
|
+
}
|
|
172
|
+
},
|
|
96
173
|
});
|
|
97
174
|
return findings;
|
|
98
175
|
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { relative } from "node:path";
|
|
2
|
+
import { classifyFieldName, isAmbiguousFieldName } from "./dataCategoryLexicon.js";
|
|
3
|
+
import { isSafeFieldName } from "./safeFieldNames.js";
|
|
4
|
+
import { redactSecrets } from "../engine/secretRedaction.js";
|
|
5
|
+
import { countConfidentSiblings } from "./confidenceFactors.js";
|
|
6
|
+
/**
|
|
7
|
+
* Flask's incoming-request accessors: request.form / request.json / request.args. Anchored to the
|
|
8
|
+
* literal name "request" rather than a configurable-name regex (unlike Express's req/request, which
|
|
9
|
+
* is a per-handler parameter that can be named anything) -- Flask's `request` is always the direct,
|
|
10
|
+
* module-level import from `flask`, used by that exact name in effectively all real code.
|
|
11
|
+
*/
|
|
12
|
+
const REQUEST_ATTR = /^request\.(form|json|args)$/;
|
|
13
|
+
const REQUEST_GET_CALL = /^request\.(form|json|args)\.get$/;
|
|
14
|
+
const REQUEST_GET_JSON_CALL = /^request\.get_json$/;
|
|
15
|
+
function stringLiteralContent(node) {
|
|
16
|
+
if (!node || node.type !== "string")
|
|
17
|
+
return null;
|
|
18
|
+
return node.descendantsOfType("string_content")[0]?.text ?? null;
|
|
19
|
+
}
|
|
20
|
+
/** True for the RHS of `x = <this>`: request.form / .json / .args, or a request.get_json() call. */
|
|
21
|
+
function isRequestDataSource(node) {
|
|
22
|
+
if (node.type === "attribute")
|
|
23
|
+
return REQUEST_ATTR.test(node.text);
|
|
24
|
+
if (node.type === "call") {
|
|
25
|
+
const fn = node.childForFieldName("function");
|
|
26
|
+
return !!fn && REQUEST_GET_JSON_CALL.test(fn.text);
|
|
27
|
+
}
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The nearest enclosing function_definition's node id, or -1 for module-level code. Scoped by `.id`
|
|
32
|
+
* rather than the Node object itself -- web-tree-sitter doesn't guarantee the same underlying syntax
|
|
33
|
+
* node comes back as the same JS object identity across separate traversal calls (same `.id`, new
|
|
34
|
+
* wrapper), so using Node references as Map keys silently never matches.
|
|
35
|
+
*/
|
|
36
|
+
function enclosingScopeId(node) {
|
|
37
|
+
let current = node.parent;
|
|
38
|
+
while (current) {
|
|
39
|
+
if (current.type === "function_definition")
|
|
40
|
+
return current.id;
|
|
41
|
+
current = current.parent;
|
|
42
|
+
}
|
|
43
|
+
return -1;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Same-function alias tracking: `data = request.get_json()` (or `= request.form` / `.json` / `.args`)
|
|
47
|
+
* followed by `data['x']` / `data.get('x')` in the same function body. Deliberately one mechanical hop,
|
|
48
|
+
* scoped to the nearest enclosing function (or module level), not general dataflow -- no cross-function
|
|
49
|
+
* tracking, no chained reassignment, no branch/conditional awareness, no closures capturing an outer
|
|
50
|
+
* function's alias. This is exactly the gap the microblog golden-corpus entry documented: real signup
|
|
51
|
+
* PII read via request.get_json() into a local variable rather than a direct request.json[...] subscript.
|
|
52
|
+
*/
|
|
53
|
+
function collectRequestAliases(tree) {
|
|
54
|
+
const aliasesByScope = new Map();
|
|
55
|
+
for (const node of tree.rootNode.descendantsOfType("assignment")) {
|
|
56
|
+
const left = node.childForFieldName("left");
|
|
57
|
+
const right = node.childForFieldName("right");
|
|
58
|
+
if (!left || left.type !== "identifier" || !right || !isRequestDataSource(right))
|
|
59
|
+
continue;
|
|
60
|
+
const scope = enclosingScopeId(node);
|
|
61
|
+
const aliases = aliasesByScope.get(scope) ?? new Set();
|
|
62
|
+
aliases.add(left.text);
|
|
63
|
+
aliasesByScope.set(scope, aliases);
|
|
64
|
+
}
|
|
65
|
+
return aliasesByScope;
|
|
66
|
+
}
|
|
67
|
+
function collectFieldAccesses(tree) {
|
|
68
|
+
const accesses = [];
|
|
69
|
+
const aliasesByScope = collectRequestAliases(tree);
|
|
70
|
+
const isAliasReference = (node) => !!node && node.type === "identifier" && !!aliasesByScope.get(enclosingScopeId(node))?.has(node.text);
|
|
71
|
+
for (const node of tree.rootNode.descendantsOfType("subscript")) {
|
|
72
|
+
const value = node.childForFieldName("value");
|
|
73
|
+
if (!value)
|
|
74
|
+
continue;
|
|
75
|
+
const isDirect = REQUEST_ATTR.test(value.text) || (value.type === "call" && isRequestDataSource(value));
|
|
76
|
+
if (!isDirect && !isAliasReference(value))
|
|
77
|
+
continue;
|
|
78
|
+
const key = stringLiteralContent(node.childForFieldName("subscript"));
|
|
79
|
+
if (key)
|
|
80
|
+
accesses.push({ fieldName: key, line: node.startPosition.row + 1 });
|
|
81
|
+
}
|
|
82
|
+
for (const node of tree.rootNode.descendantsOfType("call")) {
|
|
83
|
+
const fn = node.childForFieldName("function");
|
|
84
|
+
if (!fn)
|
|
85
|
+
continue;
|
|
86
|
+
const isDirectGet = REQUEST_GET_CALL.test(fn.text);
|
|
87
|
+
const isAliasGet = fn.type === "attribute" &&
|
|
88
|
+
fn.childForFieldName("attribute")?.text === "get" &&
|
|
89
|
+
isAliasReference(fn.childForFieldName("object"));
|
|
90
|
+
if (!isDirectGet && !isAliasGet)
|
|
91
|
+
continue;
|
|
92
|
+
const args = node.childForFieldName("arguments")?.namedChildren ?? [];
|
|
93
|
+
const key = stringLiteralContent(args[0] ?? null);
|
|
94
|
+
if (key)
|
|
95
|
+
accesses.push({ fieldName: key, line: node.startPosition.row + 1 });
|
|
96
|
+
}
|
|
97
|
+
return accesses;
|
|
98
|
+
}
|
|
99
|
+
export function runPythonFrameworkRules(tree, filePath, source, root) {
|
|
100
|
+
const lines = source.split("\n");
|
|
101
|
+
const relPath = relative(root, filePath);
|
|
102
|
+
const accesses = collectFieldAccesses(tree);
|
|
103
|
+
const allFieldNames = accesses.map((a) => a.fieldName);
|
|
104
|
+
const findings = [];
|
|
105
|
+
const emitted = new Set();
|
|
106
|
+
for (const { fieldName, line } of accesses) {
|
|
107
|
+
if (isSafeFieldName(fieldName))
|
|
108
|
+
continue;
|
|
109
|
+
const dedupeKey = `${fieldName}:${line}`;
|
|
110
|
+
if (emitted.has(dedupeKey))
|
|
111
|
+
continue;
|
|
112
|
+
emitted.add(dedupeKey);
|
|
113
|
+
const lexiconCategory = classifyFieldName(fieldName);
|
|
114
|
+
const ambiguousName = isAmbiguousFieldName(fieldName);
|
|
115
|
+
const siblingConfidentCount = countConfidentSiblings(allFieldNames, fieldName);
|
|
116
|
+
const evidenceLine = lines[line - 1] ?? "";
|
|
117
|
+
// Provisional confidence, patched by the batched protected-logic scoring call -- same pattern as
|
|
118
|
+
// frameworkRules.ts's JS equivalent; the scoring endpoint doesn't care which language a
|
|
119
|
+
// confidenceFactors-bearing finding came from.
|
|
120
|
+
findings.push({
|
|
121
|
+
id: `field:${relPath}:${line}:${fieldName}`,
|
|
122
|
+
dataCategories: lexiconCategory ? [lexiconCategory] : ["generic_pii"],
|
|
123
|
+
processor: null,
|
|
124
|
+
description: lexiconCategory
|
|
125
|
+
? `Request field "${fieldName}" collected, classified as ${lexiconCategory}.`
|
|
126
|
+
: `Request field "${fieldName}" collected; could not confidently classify — needs review.`,
|
|
127
|
+
confidence: "low",
|
|
128
|
+
source: "framework-rule",
|
|
129
|
+
location: { file: relPath, line, column: 0 },
|
|
130
|
+
evidence: redactSecrets(evidenceLine.trim()).slice(0, 200),
|
|
131
|
+
requiresReview: true,
|
|
132
|
+
confidenceFactors: { lexiconCategory, ambiguousName, runtimeVerified: true, siblingConfidentCount },
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
return findings;
|
|
136
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { relative } from "node:path";
|
|
2
|
+
import { PROCESSOR_PROFILES } from "./processorProfiles.js";
|
|
3
|
+
import { redactSecrets } from "../engine/secretRedaction.js";
|
|
4
|
+
/**
|
|
5
|
+
* Deliberately narrower than sdkRules.ts's JS table -- not every processor has a reliable
|
|
6
|
+
* import-name signal in Python's ecosystem:
|
|
7
|
+
* - Cognito: Python code almost always calls it through boto3, AWS's *general* SDK used for every
|
|
8
|
+
* AWS service (S3, DynamoDB, SQS, ...). Mapping a bare `import boto3` to "Amazon Cognito" would be
|
|
9
|
+
* wrong far more often than right; reliably detecting it needs finding the specific
|
|
10
|
+
* `boto3.client("cognito-idp")` call, not just the import -- a real gap, not covered here.
|
|
11
|
+
* - Google Analytics: no single well-known, unambiguous pip import root the way `stripe` or
|
|
12
|
+
* `twilio` are for their processors. Left out rather than guessed.
|
|
13
|
+
*/
|
|
14
|
+
const SDK_MATCHERS = [
|
|
15
|
+
{ processorKey: "stripe", moduleNames: ["stripe"] },
|
|
16
|
+
{ processorKey: "sendgrid", moduleNames: ["sendgrid"] },
|
|
17
|
+
{ processorKey: "twilio", moduleNames: ["twilio"] },
|
|
18
|
+
{ processorKey: "mixpanel", moduleNames: ["mixpanel"] },
|
|
19
|
+
{ processorKey: "auth0", moduleNames: ["auth0"] },
|
|
20
|
+
// PyPI package is "firebase-admin"; its import root is firebase_admin.
|
|
21
|
+
{ processorKey: "firebase", moduleNames: ["firebase_admin"] },
|
|
22
|
+
];
|
|
23
|
+
function matchProcessor(moduleRoot) {
|
|
24
|
+
return SDK_MATCHERS.find((matcher) => matcher.moduleNames.includes(moduleRoot));
|
|
25
|
+
}
|
|
26
|
+
/** The root package name from a dotted module path node (e.g. "twilio" from "twilio.rest"). */
|
|
27
|
+
function moduleRootName(moduleNameNode) {
|
|
28
|
+
if (!moduleNameNode)
|
|
29
|
+
return null;
|
|
30
|
+
if (moduleNameNode.type === "dotted_name") {
|
|
31
|
+
return moduleNameNode.child(0)?.text ?? null;
|
|
32
|
+
}
|
|
33
|
+
if (moduleNameNode.type === "aliased_import") {
|
|
34
|
+
return moduleRootName(moduleNameNode.childForFieldName("name"));
|
|
35
|
+
}
|
|
36
|
+
// relative_import ("from . import x") and anything else has no external package name.
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
export function runPythonSdkRules(tree, filePath, source, root) {
|
|
40
|
+
const findings = [];
|
|
41
|
+
const lines = source.split("\n");
|
|
42
|
+
const relPath = relative(root, filePath);
|
|
43
|
+
const seenProcessors = new Set();
|
|
44
|
+
const importNodes = [
|
|
45
|
+
...tree.rootNode.descendantsOfType("import_statement"),
|
|
46
|
+
...tree.rootNode.descendantsOfType("import_from_statement"),
|
|
47
|
+
];
|
|
48
|
+
for (const node of importNodes) {
|
|
49
|
+
const fieldName = node.type === "import_statement" ? "name" : "module_name";
|
|
50
|
+
const moduleRoot = moduleRootName(node.childForFieldName(fieldName));
|
|
51
|
+
if (!moduleRoot)
|
|
52
|
+
continue;
|
|
53
|
+
const matcher = matchProcessor(moduleRoot);
|
|
54
|
+
if (!matcher || seenProcessors.has(matcher.processorKey))
|
|
55
|
+
continue;
|
|
56
|
+
seenProcessors.add(matcher.processorKey);
|
|
57
|
+
const profile = PROCESSOR_PROFILES[matcher.processorKey];
|
|
58
|
+
const line = node.startPosition.row + 1;
|
|
59
|
+
const evidenceLine = lines[line - 1] ?? "";
|
|
60
|
+
findings.push({
|
|
61
|
+
id: `sdk:${matcher.processorKey}:${relPath}:${line}`,
|
|
62
|
+
dataCategories: profile.dataCategories,
|
|
63
|
+
processor: profile.name,
|
|
64
|
+
description: `${profile.name} integration detected. ${profile.disclosureNotes}`,
|
|
65
|
+
confidence: "high",
|
|
66
|
+
source: "sdk-rule",
|
|
67
|
+
location: { file: relPath, line, column: 0 },
|
|
68
|
+
evidence: redactSecrets(evidenceLine.trim()),
|
|
69
|
+
requiresReview: false,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
return findings;
|
|
73
|
+
}
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
// Exact-match only, deliberately narrow -- this is the one silent-drop path in the whole pipeline
|
|
2
|
+
// (a name on this list never becomes a finding at all, not even a low-confidence one flagged for
|
|
3
|
+
// review). Names that are ALSO plausible PII in some real API (e.g. "next"/"prev" as redirect
|
|
4
|
+
// targets, "since"/"until"/"before"/"after" as ambiguous short words) are deliberately left off even
|
|
5
|
+
// when they're common pagination/cursor names elsewhere, so they stay visible for manual review
|
|
6
|
+
// rather than being silently and permanently suppressed.
|
|
1
7
|
const SAFE_FIELD_NAMES = new Set([
|
|
2
8
|
"id",
|
|
3
9
|
"uuid",
|
|
@@ -16,6 +22,14 @@ const SAFE_FIELD_NAMES = new Set([
|
|
|
16
22
|
"total",
|
|
17
23
|
"sort",
|
|
18
24
|
"order",
|
|
25
|
+
"perpage",
|
|
26
|
+
"pagesize",
|
|
27
|
+
"cursor",
|
|
28
|
+
"nextcursor",
|
|
29
|
+
"prevcursor",
|
|
30
|
+
"sortby",
|
|
31
|
+
"orderby",
|
|
32
|
+
"sortorder",
|
|
19
33
|
]);
|
|
20
34
|
function normalize(fieldName) {
|
|
21
35
|
return fieldName.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A hard ceiling, not a normal operating limit: protectedLogicClient.ts batches by total bytes, so a
|
|
3
|
+
* large fragment is sent ALONE in its own request rather than bundled with others -- the actual
|
|
4
|
+
* per-batch CPU risk that caused intermittent "Worker exceeded CPU time limit" failures (confirmed via
|
|
5
|
+
* wrangler tail) is handled there, not here. This constant exists only to refuse something pathological
|
|
6
|
+
* (e.g. a machine-generated file with one enormous schema), not to cap ordinary real-world files -- and
|
|
7
|
+
* trimmed extraction (above) means ordinary files need it even less than before.
|
|
8
|
+
*
|
|
9
|
+
* Measured (uncapped, pre-trimming) across the three golden-corpus repos: max 41,992 bytes
|
|
10
|
+
* (formbricks/packages/types/surveys/types.ts). This value is PROVISIONAL and unvalidated against the
|
|
11
|
+
* real Worker limit: that depends on a Cloudflare plan upgrade that hasn't happened yet. Revisit once
|
|
12
|
+
* the upgrade is confirmed and the 10-run wrangler tail test has been repeated.
|
|
13
|
+
*/
|
|
14
|
+
export declare const MAX_FRAGMENT_BYTES = 65536;
|
|
1
15
|
export interface ZodCandidate {
|
|
2
16
|
id: string;
|
|
3
17
|
codeFragment: string;
|
|
@@ -6,4 +20,10 @@ export interface ZodCandidate {
|
|
|
6
20
|
* since the fragment is a non-contiguous excerpt of the original file, not a raw slice. */
|
|
7
21
|
fragmentLineToSourceLine: number[];
|
|
8
22
|
}
|
|
9
|
-
export
|
|
23
|
+
export interface FindZodCandidatesResult {
|
|
24
|
+
candidates: ZodCandidate[];
|
|
25
|
+
/** Relative paths of files whose extracted fragment exceeded MAX_FRAGMENT_BYTES and was dropped
|
|
26
|
+
* rather than sent -- the caller must surface these, never drop them silently. */
|
|
27
|
+
oversizedFiles: string[];
|
|
28
|
+
}
|
|
29
|
+
export declare function findZodCandidates(source: string, filePath: string, root: string): FindZodCandidatesResult;
|