srclawyer 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/cli.js +45 -3
- package/dist/commands/init.d.ts +16 -1
- package/dist/commands/init.js +116 -17
- package/dist/commands/scan.js +12 -2
- package/dist/config/config.d.ts +12 -1
- package/dist/config/config.js +12 -1
- package/dist/engine/discoverFiles.d.ts +2 -1
- package/dist/engine/discoverFiles.js +7 -7
- package/dist/engine/discoverProjects.d.ts +9 -1
- package/dist/engine/discoverProjects.js +71 -18
- package/dist/engine/globFiles.d.ts +10 -0
- package/dist/engine/globFiles.js +12 -0
- package/dist/engine/languageDetection.d.ts +2 -1
- package/dist/engine/languageDetection.js +10 -10
- package/dist/engine/rootBoundary.d.ts +29 -0
- package/dist/engine/rootBoundary.js +102 -0
- package/dist/engine/scanEngine.js +174 -58
- package/dist/engine/scanLimits.d.ts +29 -0
- package/dist/engine/scanLimits.js +70 -0
- package/dist/engine/tsProject.d.ts +2 -1
- package/dist/engine/tsProject.js +40 -3
- package/dist/engine/types.d.ts +16 -0
- package/dist/engine/workspacePatterns.d.ts +23 -0
- package/dist/engine/workspacePatterns.js +140 -0
- package/dist/policy/generatePolicy.js +143 -3
- package/dist/report/formatText.js +16 -0
- package/dist/rules/htmlRules.d.ts +11 -0
- package/dist/rules/htmlRules.js +73 -17
- package/dist/rules/zodPreFilter.js +10 -1
- package/dist/schemaParsers/openapi.d.ts +6 -1
- package/dist/schemaParsers/openapi.js +3 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -90,6 +90,8 @@ Every finding traces back to a file and line (`src/engine/types.ts#Finding`).
|
|
|
90
90
|
- **Data sent to a third party via a raw HTTP call isn't attributed to that processor.** SDK-integration detection (above) is anchored to known package imports. A call like Python's `requests.post("https://api.some-provider.com/...", json={...})` or JS's `fetch("https://api.some-provider.com/...")` — bypassing a dedicated SDK entirely — is invisible to this detection today, in both languages, regardless of how sensitive the data it sends is.
|
|
91
91
|
- **Python support is based on one real-world corpus repository so far** (a Flask app; see `test-fixtures/golden-corpus/`), not the broader validation JS detection has had. Expect rougher edges on Flask codebases with unusual patterns.
|
|
92
92
|
- **OpenAPI/Swagger scanning reports every schema under `components.schemas`, including response-only ones** — a schema that's only ever returned by the server (never submitted by a client) gets identical treatment to a real request-input schema, since nothing currently checks whether a schema is actually reachable from an operation's `requestBody` or `parameters`. On a large, schema-heavy spec this can mean a long "Items Requiring Manual Review" list that includes fields the product never collects, only ever returns. Separately, a schema defined **inline** inside a `requestBody` (not as a named, `$ref`'d `components.schemas` entry) isn't read at all today, in either direction — only named component schemas are scanned.
|
|
93
|
+
- **Flask support covers direct request reads and same-function aliases only.** It detects `request.form[...]`, `request.args.get(...)`, `request.json[...]` and the like, plus one assignment hop within a function (`data = request.get_json()`, then `data['x']`). Form classes — Flask-WTF/WTForms `FlaskForm` subclasses with `StringField`/`PasswordField` declarations read via `form.email.data` — are not detected, and neither are fields rendered by template helpers such as `{{ form.email() }}`. A Flask app that collects data through WTForms may therefore show fewer collected fields than it actually has.
|
|
94
|
+
- **Server-side template syntax inside HTML is not analyzed.** Jinja, Django, ERB and similar delimiters (`{{`, `{%`, `<%`, `{#`) are not interpreted: an inline `<script>` that isn't valid JavaScript until the template engine runs is skipped (listed in `--json` as `template-syntax`, with a one-line note), and form fields that a template generates rather than spells out as `<input>` tags are not seen.
|
|
93
95
|
|
|
94
96
|
## Network calls
|
|
95
97
|
|
package/dist/cli.js
CHANGED
|
@@ -16,9 +16,51 @@ program
|
|
|
16
16
|
.version(version);
|
|
17
17
|
program
|
|
18
18
|
.command("init")
|
|
19
|
-
.description("One-time setup: jurisdiction, markets, industry")
|
|
20
|
-
.
|
|
21
|
-
|
|
19
|
+
.description("One-time setup: jurisdiction, markets, industry, and optional policy details (company info, data practices, regional rights)")
|
|
20
|
+
.option("--policy-details", "ask the optional policy-detail questions (company info, data practices, regional rights)")
|
|
21
|
+
.option("--no-policy-details", "skip all optional policy-detail questions, leaving them unanswered")
|
|
22
|
+
.option("--collects-childrens-data", "declare that this product knowingly collects data from children under 13/16")
|
|
23
|
+
.option("--no-collects-childrens-data", "declare that this product does not knowingly collect data from children under 13/16")
|
|
24
|
+
.option("--company-name <name>", "company or product name")
|
|
25
|
+
.option("--site-url <url>", "site or app URL")
|
|
26
|
+
.option("--privacy-contact-email <email>", "privacy contact email address")
|
|
27
|
+
.option("--eu-uk-users", "declare that this product has users in the EU/UK")
|
|
28
|
+
.option("--no-eu-uk-users", "declare that this product has no users in the EU/UK")
|
|
29
|
+
.option("--california-users", "declare that this product has users in California")
|
|
30
|
+
.option("--no-california-users", "declare that this product has no users in California")
|
|
31
|
+
.option("--sells-or-shares-data", "declare that personal data is sold or shared with third parties")
|
|
32
|
+
.option("--no-sells-or-shares-data", "declare that personal data is not sold or shared with third parties")
|
|
33
|
+
.option("--uses-data-for-advertising", "declare that personal data is used for advertising")
|
|
34
|
+
.option("--no-uses-data-for-advertising", "declare that personal data is not used for advertising")
|
|
35
|
+
.option("--data-retention-policy <text>", "how long personal data is retained")
|
|
36
|
+
.option("--account-deletion-instructions <text>", "how a user can delete their account or request data deletion")
|
|
37
|
+
.option("--minimum-age <number>", "minimum age required to use this product")
|
|
38
|
+
.option("--sends-data-to-ai-services", "declare that user data is sent to third-party AI services")
|
|
39
|
+
.option("--no-sends-data-to-ai-services", "declare that user data is not sent to third-party AI services")
|
|
40
|
+
.action(async (opts) => {
|
|
41
|
+
let minimumAge;
|
|
42
|
+
if (opts.minimumAge !== undefined) {
|
|
43
|
+
const parsed = Number(opts.minimumAge);
|
|
44
|
+
if (!Number.isInteger(parsed) || parsed < 0) {
|
|
45
|
+
program.error(`error: --minimum-age must be a whole, non-negative number (got "${opts.minimumAge}")`);
|
|
46
|
+
}
|
|
47
|
+
minimumAge = parsed;
|
|
48
|
+
}
|
|
49
|
+
await runInit(process.cwd(), {
|
|
50
|
+
policyDetails: opts.policyDetails,
|
|
51
|
+
collectsChildrensData: opts.collectsChildrensData,
|
|
52
|
+
companyName: opts.companyName,
|
|
53
|
+
siteUrl: opts.siteUrl,
|
|
54
|
+
privacyContactEmail: opts.privacyContactEmail,
|
|
55
|
+
euUkUsers: opts.euUkUsers,
|
|
56
|
+
californiaUsers: opts.californiaUsers,
|
|
57
|
+
sellsOrSharesData: opts.sellsOrSharesData,
|
|
58
|
+
usesDataForAdvertising: opts.usesDataForAdvertising,
|
|
59
|
+
dataRetentionPolicy: opts.dataRetentionPolicy,
|
|
60
|
+
accountDeletionInstructions: opts.accountDeletionInstructions,
|
|
61
|
+
minimumAge,
|
|
62
|
+
sendsDataToAiServices: opts.sendsDataToAiServices,
|
|
63
|
+
});
|
|
22
64
|
});
|
|
23
65
|
program
|
|
24
66
|
.command("scan")
|
package/dist/commands/init.d.ts
CHANGED
|
@@ -1 +1,16 @@
|
|
|
1
|
-
export
|
|
1
|
+
export interface InitFlags {
|
|
2
|
+
policyDetails?: boolean;
|
|
3
|
+
collectsChildrensData?: boolean;
|
|
4
|
+
companyName?: string;
|
|
5
|
+
siteUrl?: string;
|
|
6
|
+
privacyContactEmail?: string;
|
|
7
|
+
euUkUsers?: boolean;
|
|
8
|
+
californiaUsers?: boolean;
|
|
9
|
+
sellsOrSharesData?: boolean;
|
|
10
|
+
usesDataForAdvertising?: boolean;
|
|
11
|
+
dataRetentionPolicy?: string;
|
|
12
|
+
accountDeletionInstructions?: string;
|
|
13
|
+
minimumAge?: number;
|
|
14
|
+
sendsDataToAiServices?: boolean;
|
|
15
|
+
}
|
|
16
|
+
export declare function runInit(root: string, flags?: InitFlags): Promise<void>;
|
package/dist/commands/init.js
CHANGED
|
@@ -3,6 +3,7 @@ import { join } from "node:path";
|
|
|
3
3
|
import { createInterface } from "node:readline/promises";
|
|
4
4
|
import { CONFIG_FILENAME, DEFAULT_CONFIG, configPath, hasUnmigratedLegacyConfig, legacyConfigMessage, writeConfig, } from "../config/config.js";
|
|
5
5
|
import { NETWORK_CALLS_NOTICE } from "../policy/legalNotices.js";
|
|
6
|
+
import { runScan } from "./scan.js";
|
|
6
7
|
function guessIndustry(root) {
|
|
7
8
|
const pkgPath = join(root, "package.json");
|
|
8
9
|
if (!existsSync(pkgPath))
|
|
@@ -24,16 +25,76 @@ async function ask(lines, question, fallback) {
|
|
|
24
25
|
const answer = done ? "" : value.trim();
|
|
25
26
|
return answer.length > 0 ? answer : fallback;
|
|
26
27
|
}
|
|
27
|
-
|
|
28
|
+
async function askOptional(lines, question) {
|
|
29
|
+
process.stdout.write(`${question} (optional, press Enter to skip): `);
|
|
30
|
+
const { value, done } = await lines.next();
|
|
31
|
+
const answer = done ? "" : value.trim();
|
|
32
|
+
return answer.length > 0 ? answer : null;
|
|
33
|
+
}
|
|
34
|
+
// suggested, when not null, is shown as the bracketed default and adopted on a blank answer -- used
|
|
35
|
+
// to prefill EU/UK and California questions from targetMarkets without ever forcing an answer. When
|
|
36
|
+
// suggested is null (no signal), blank means genuinely unanswered, same as every other optional field.
|
|
37
|
+
async function askYesNoOptional(lines, question, suggested = null) {
|
|
38
|
+
const suffix = suggested === null ? " (y/n, optional, press Enter to skip)" : ` [${suggested ? "y" : "n"}]`;
|
|
39
|
+
process.stdout.write(`${question}${suffix}: `);
|
|
40
|
+
const { value, done } = await lines.next();
|
|
41
|
+
const answer = done ? "" : value.trim().toLowerCase();
|
|
42
|
+
if (answer.length === 0)
|
|
43
|
+
return suggested;
|
|
44
|
+
return /^y(es)?$/.test(answer);
|
|
45
|
+
}
|
|
46
|
+
async function askOptionalAge(lines, question) {
|
|
47
|
+
const raw = await askOptional(lines, question);
|
|
48
|
+
if (raw === null)
|
|
49
|
+
return null;
|
|
50
|
+
const parsed = Number(raw);
|
|
51
|
+
if (!Number.isInteger(parsed) || parsed < 0) {
|
|
52
|
+
console.log(` (Could not read "${raw}" as a whole, non-negative number -- skipping. You can set this later by editing ${CONFIG_FILENAME}.)`);
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
return parsed;
|
|
56
|
+
}
|
|
57
|
+
// Generic resolution order for every new optional field: an explicit flag always wins outright;
|
|
58
|
+
// otherwise the field is only asked if the policy-details gate resolved to yes; otherwise it's null.
|
|
59
|
+
async function resolveField(flagValue, gateYes, askFn) {
|
|
60
|
+
if (flagValue !== undefined)
|
|
61
|
+
return flagValue;
|
|
62
|
+
if (!gateYes)
|
|
63
|
+
return null;
|
|
64
|
+
return askFn();
|
|
65
|
+
}
|
|
66
|
+
const EU_UK_TOKENS = new Set(["eu", "uk", "gb", "eea", "european union", "united kingdom"]);
|
|
67
|
+
const CALIFORNIA_TOKENS = new Set(["ca", "california", "us-ca"]);
|
|
68
|
+
function targetMarketsSuggestEuUk(markets) {
|
|
69
|
+
return markets.some((m) => EU_UK_TOKENS.has(m.trim().toLowerCase()));
|
|
70
|
+
}
|
|
71
|
+
function targetMarketsSuggestCalifornia(markets) {
|
|
72
|
+
return markets.some((m) => CALIFORNIA_TOKENS.has(m.trim().toLowerCase()));
|
|
73
|
+
}
|
|
74
|
+
// Only warns in the one direction that's a real contradiction: target markets signal a region, but
|
|
75
|
+
// the final answer explicitly denies it. The reverse (no signal, but the user says yes) isn't a
|
|
76
|
+
// contradiction -- targetMarkets is free text and was never meant to be an exhaustive jurisdiction
|
|
77
|
+
// list. Never blocks or overrides -- the explicit answer always stands, this only warns.
|
|
78
|
+
function warnIfContradictsTargetMarkets(label, finalValue, suggestedFromMarkets) {
|
|
79
|
+
if (suggestedFromMarkets && finalValue === false) {
|
|
80
|
+
console.log(`\nWarning: your target markets suggest ${label} users, but you indicated none -- the generated policy will omit ${label}-specific disclosures. Re-run init or edit ${CONFIG_FILENAME} if this is incorrect.\n`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
export async function runInit(root, flags = {}) {
|
|
28
84
|
const existingPath = configPath(root);
|
|
29
85
|
if (existsSync(existingPath)) {
|
|
30
|
-
console.log(`${CONFIG_FILENAME} already exists.
|
|
86
|
+
console.log(`${CONFIG_FILENAME} already exists -- left unchanged. To start over, delete it and re-run "srclawyer init":\n\n rm ${CONFIG_FILENAME}\n srclawyer init\n`);
|
|
31
87
|
return;
|
|
32
88
|
}
|
|
33
89
|
if (hasUnmigratedLegacyConfig(root)) {
|
|
34
90
|
console.log(legacyConfigMessage());
|
|
35
91
|
return;
|
|
36
92
|
}
|
|
93
|
+
// The final "run a scan now?" prompt below is interactive-only: skipped whenever this run is
|
|
94
|
+
// flag-driven (any flag was passed -- a scripted/CI invocation, even a partial one) or stdin isn't
|
|
95
|
+
// a real TTY (piped, redirected, non-interactive). Computed once, up front, rather than re-checked
|
|
96
|
+
// at the end, since nothing between here and there changes either condition.
|
|
97
|
+
const isInteractive = process.stdin.isTTY === true && Object.values(flags).every((v) => v === undefined);
|
|
37
98
|
const guessedIndustry = guessIndustry(root);
|
|
38
99
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
39
100
|
const lines = rl[Symbol.asyncIterator]();
|
|
@@ -43,37 +104,75 @@ export async function runInit(root) {
|
|
|
43
104
|
console.log("SrcLawyer setup — answered once, reused on every scan.\n");
|
|
44
105
|
const entityLocation = await ask(lines, "Where is your legal entity located? (country)", DEFAULT_CONFIG.entityLocation ?? "");
|
|
45
106
|
const targetMarketsRaw = await ask(lines, "Which regions/markets do you serve? (comma-separated, e.g. US, EU)", "");
|
|
107
|
+
const targetMarkets = targetMarketsRaw.split(",").map((s) => s.trim()).filter(Boolean);
|
|
46
108
|
const industry = await ask(lines, "What industry/vertical is this product in?", guessedIndustry ?? "");
|
|
47
|
-
const
|
|
109
|
+
const gateYes = flags.policyDetails !== undefined
|
|
110
|
+
? flags.policyDetails
|
|
111
|
+
: /^y(es)?$/i.test((await ask(lines, "Add privacy policy details now? (company info, data practices, regional rights, etc.)", "y")).trim());
|
|
112
|
+
// Deliberately requires an explicit y/n -- a bare Enter, empty stdin, or a declined gate must leave
|
|
113
|
+
// this null (unanswered, rendered as a placeholder), never silently "no". Unlike askOptional's other
|
|
114
|
+
// callers, there is no safe default here: "no" is itself a real, consequential legal declaration, not
|
|
115
|
+
// a neutral fallback. Layer 2's high-stakes override (scan.ts) coerces null to false at its one call
|
|
116
|
+
// site, so leaving this unanswered never weakens that safety check -- see childrensDataOverride.test.ts.
|
|
117
|
+
const collectsChildrensData = await resolveField(flags.collectsChildrensData, gateYes, () => askYesNoOptional(lines, "Does this product knowingly collect data from children under 13/16?"));
|
|
118
|
+
const minimumAge = await resolveField(flags.minimumAge, gateYes, () => askOptionalAge(lines, "Minimum age required to use this product"));
|
|
119
|
+
const companyName = await resolveField(flags.companyName, gateYes, () => askOptional(lines, "Company or product name"));
|
|
120
|
+
const siteUrl = await resolveField(flags.siteUrl, gateYes, () => askOptional(lines, "Site or app URL"));
|
|
121
|
+
const privacyContactEmail = await resolveField(flags.privacyContactEmail, gateYes, () => askOptional(lines, "Privacy contact email"));
|
|
122
|
+
const euUkSuggested = targetMarketsSuggestEuUk(targetMarkets);
|
|
123
|
+
const hasEuUkUsers = await resolveField(flags.euUkUsers, gateYes, () => askYesNoOptional(lines, "Do you have users in the EU/UK?", euUkSuggested ? true : null));
|
|
124
|
+
warnIfContradictsTargetMarkets("EU/UK", hasEuUkUsers, euUkSuggested);
|
|
125
|
+
const californiaSuggested = targetMarketsSuggestCalifornia(targetMarkets);
|
|
126
|
+
const hasCaliforniaUsers = await resolveField(flags.californiaUsers, gateYes, () => askYesNoOptional(lines, "Do you have users in California?", californiaSuggested ? true : null));
|
|
127
|
+
warnIfContradictsTargetMarkets("California", hasCaliforniaUsers, californiaSuggested);
|
|
128
|
+
const sellsOrSharesData = await resolveField(flags.sellsOrSharesData, gateYes, () => askYesNoOptional(lines, "Does this product sell or share personal data with third parties?"));
|
|
129
|
+
const usesDataForAdvertising = await resolveField(flags.usesDataForAdvertising, gateYes, () => askYesNoOptional(lines, "Does this product use personal data for advertising?"));
|
|
130
|
+
const dataRetentionPolicy = await resolveField(flags.dataRetentionPolicy, gateYes, () => askOptional(lines, "How long is personal data retained?"));
|
|
131
|
+
const accountDeletionInstructions = await resolveField(flags.accountDeletionInstructions, gateYes, () => askOptional(lines, "How can a user delete their account or request data deletion?"));
|
|
132
|
+
// Deliberately about the SCANNED PRODUCT's own behavior (does it send its end-users' data to AI
|
|
133
|
+
// services as part of its own functionality) -- not about SrcLawyer's own, separately-documented
|
|
134
|
+
// BYOK network calls described in NETWORK_CALLS_NOTICE above.
|
|
135
|
+
const sendsDataToAiServices = await resolveField(flags.sendsDataToAiServices, gateYes, () => askYesNoOptional(lines, "Does this product send user data to third-party AI services (e.g. OpenAI, Anthropic) as part of its own functionality?"));
|
|
48
136
|
// "managed" is deliberately not offered here -- it's a real SrcLawyerTier value and config.ts/
|
|
49
137
|
// resolveProviderForTier.ts both still support it fully (a hand-edited config with tier: managed
|
|
50
138
|
// still gets a clear, accurate error at scan time), but there's no advertised path to it from this
|
|
51
139
|
// prompt, so a tester reading the choices here can't select a dead end during setup.
|
|
52
|
-
const
|
|
53
|
-
const tier =
|
|
140
|
+
const enableByokRaw = await ask(lines, "Enable optional AI-assisted review with your own Anthropic key? (advanced)", "n");
|
|
141
|
+
const tier = /^y(es)?$/i.test(enableByokRaw.trim()) ? "byok" : "free";
|
|
54
142
|
let apiKeyEnvVar = null;
|
|
55
143
|
if (tier === "byok") {
|
|
56
144
|
apiKeyEnvVar = await ask(lines, "Which environment variable holds your Anthropic API key?", "ANTHROPIC_API_KEY");
|
|
57
145
|
console.log(`\nNote: only the name "${apiKeyEnvVar}" is written to ${CONFIG_FILENAME} — the key itself is never written to disk. It's read from your environment at scan time.`);
|
|
58
146
|
}
|
|
59
|
-
else if (tier === "managed") {
|
|
60
|
-
console.log("\nHeads up: the managed tier's hosted backend isn't available yet. Layer 1 (static analysis) will still run fine; AI-assisted resolution will report a clear error until it ships.");
|
|
61
|
-
}
|
|
62
|
-
rl.close();
|
|
63
147
|
const config = {
|
|
64
148
|
entityLocation: entityLocation || null,
|
|
65
|
-
targetMarkets
|
|
149
|
+
targetMarkets,
|
|
66
150
|
industry: industry || null,
|
|
67
|
-
collectsChildrensData
|
|
151
|
+
collectsChildrensData,
|
|
68
152
|
tier,
|
|
69
153
|
...(tier === "byok" ? { llm: { provider: "anthropic", apiKeyEnvVar } } : {}),
|
|
154
|
+
companyName,
|
|
155
|
+
siteUrl,
|
|
156
|
+
privacyContactEmail,
|
|
157
|
+
hasEuUkUsers,
|
|
158
|
+
hasCaliforniaUsers,
|
|
159
|
+
sellsOrSharesData,
|
|
160
|
+
usesDataForAdvertising,
|
|
161
|
+
dataRetentionPolicy,
|
|
162
|
+
accountDeletionInstructions,
|
|
163
|
+
minimumAge,
|
|
164
|
+
sendsDataToAiServices,
|
|
70
165
|
};
|
|
71
166
|
writeConfig(root, config);
|
|
72
167
|
console.log(`\nWrote ${CONFIG_FILENAME}. Run "srclawyer scan" to analyze this codebase.`);
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
168
|
+
let runScanNow = false;
|
|
169
|
+
if (isInteractive) {
|
|
170
|
+
const raw = await ask(lines, "Run a scan now and generate your policy?", "y");
|
|
171
|
+
runScanNow = /^y(es)?$/i.test(raw.trim());
|
|
172
|
+
}
|
|
173
|
+
rl.close();
|
|
174
|
+
if (runScanNow) {
|
|
175
|
+
console.log("");
|
|
176
|
+
await runScan(root, { json: false, out: "PRIVACY_POLICY.md", llm: true });
|
|
177
|
+
}
|
|
79
178
|
}
|
package/dist/commands/scan.js
CHANGED
|
@@ -24,6 +24,12 @@ export async function runScan(root, options) {
|
|
|
24
24
|
}
|
|
25
25
|
const result = await scan(root);
|
|
26
26
|
let findings = result.findings;
|
|
27
|
+
if (result.workspaceWarning)
|
|
28
|
+
console.error(result.workspaceWarning);
|
|
29
|
+
if (result.skippedFilesWarning)
|
|
30
|
+
console.error(result.skippedFilesWarning);
|
|
31
|
+
if (result.templateSyntaxNote)
|
|
32
|
+
console.error(result.templateSyntaxNote);
|
|
27
33
|
if (result.maxZodFragmentBytes > 0) {
|
|
28
34
|
console.error(`Largest Zod-schema fragment this scan: ${result.maxZodFragmentBytes} bytes (hard ceiling: ${MAX_FRAGMENT_BYTES} bytes).`);
|
|
29
35
|
}
|
|
@@ -56,7 +62,11 @@ export async function runScan(root, options) {
|
|
|
56
62
|
}
|
|
57
63
|
console.log(`Sending ${ambiguous.length} ambiguous item(s) to ${provider.name}...`);
|
|
58
64
|
const resolved = await resolveAmbiguousFindings(ambiguous, provider, {
|
|
59
|
-
|
|
65
|
+
// Preserves today's exact override behavior bit-for-bit: a missing/legacy key already
|
|
66
|
+
// behaved as false before collectsChildrensData became nullable. Only the generated
|
|
67
|
+
// document (generatePolicy.ts) distinguishes "explicitly no" from "never answered" -- this
|
|
68
|
+
// safety-critical override is unchanged.
|
|
69
|
+
collectsChildrensData: config.collectsChildrensData ?? false,
|
|
60
70
|
});
|
|
61
71
|
const resolvedById = new Map(resolved.map((f) => [f.id, f]));
|
|
62
72
|
findings = findings.map((f) => resolvedById.get(f.id) ?? f);
|
|
@@ -67,7 +77,7 @@ export async function runScan(root, options) {
|
|
|
67
77
|
const finalResult = { ...result, findings, ambiguousCount };
|
|
68
78
|
const policyPath = resolve(root, options.out);
|
|
69
79
|
writeFileSync(policyPath, generatePolicyMarkdown(finalResult, config, llmError, protectedLogicWarning));
|
|
70
|
-
if (result.unsupportedStackWarning || llmError || protectedLogicWarning || legacyConfigWarning) {
|
|
80
|
+
if (result.unsupportedStackWarning || result.workspaceWarning || result.skippedFilesWarning || llmError || protectedLogicWarning || legacyConfigWarning) {
|
|
71
81
|
process.exitCode = 1;
|
|
72
82
|
}
|
|
73
83
|
if (options.json) {
|
package/dist/config/config.d.ts
CHANGED
|
@@ -10,9 +10,20 @@ export interface SrcLawyerConfig {
|
|
|
10
10
|
entityLocation: string | null;
|
|
11
11
|
targetMarkets: string[];
|
|
12
12
|
industry: string | null;
|
|
13
|
-
collectsChildrensData: boolean;
|
|
13
|
+
collectsChildrensData: boolean | null;
|
|
14
14
|
tier?: SrcLawyerTier;
|
|
15
15
|
llm?: SrcLawyerLlmConfig;
|
|
16
|
+
companyName?: string | null;
|
|
17
|
+
siteUrl?: string | null;
|
|
18
|
+
privacyContactEmail?: string | null;
|
|
19
|
+
hasEuUkUsers?: boolean | null;
|
|
20
|
+
hasCaliforniaUsers?: boolean | null;
|
|
21
|
+
sellsOrSharesData?: boolean | null;
|
|
22
|
+
usesDataForAdvertising?: boolean | null;
|
|
23
|
+
dataRetentionPolicy?: string | null;
|
|
24
|
+
accountDeletionInstructions?: string | null;
|
|
25
|
+
minimumAge?: number | null;
|
|
26
|
+
sendsDataToAiServices?: boolean | null;
|
|
16
27
|
}
|
|
17
28
|
export declare const DEFAULT_CONFIG: SrcLawyerConfig;
|
|
18
29
|
export declare function configPath(root: string): string;
|
package/dist/config/config.js
CHANGED
|
@@ -8,8 +8,19 @@ export const DEFAULT_CONFIG = {
|
|
|
8
8
|
entityLocation: null,
|
|
9
9
|
targetMarkets: [],
|
|
10
10
|
industry: null,
|
|
11
|
-
collectsChildrensData:
|
|
11
|
+
collectsChildrensData: null,
|
|
12
12
|
tier: "free",
|
|
13
|
+
companyName: null,
|
|
14
|
+
siteUrl: null,
|
|
15
|
+
privacyContactEmail: null,
|
|
16
|
+
hasEuUkUsers: null,
|
|
17
|
+
hasCaliforniaUsers: null,
|
|
18
|
+
sellsOrSharesData: null,
|
|
19
|
+
usesDataForAdvertising: null,
|
|
20
|
+
dataRetentionPolicy: null,
|
|
21
|
+
accountDeletionInstructions: null,
|
|
22
|
+
minimumAge: null,
|
|
23
|
+
sendsDataToAiServices: null,
|
|
13
24
|
};
|
|
14
25
|
export function configPath(root) {
|
|
15
26
|
return join(root, CONFIG_FILENAME);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { RootBoundary } from "./rootBoundary.js";
|
|
1
2
|
export interface DiscoveredFiles {
|
|
2
3
|
codeFiles: string[];
|
|
3
4
|
pythonFiles: string[];
|
|
@@ -5,4 +6,4 @@ export interface DiscoveredFiles {
|
|
|
5
6
|
schemaFiles: string[];
|
|
6
7
|
manifestFiles: string[];
|
|
7
8
|
}
|
|
8
|
-
export declare function discoverFiles(root: string): Promise<DiscoveredFiles>;
|
|
9
|
+
export declare function discoverFiles(root: string, boundary: RootBoundary): Promise<DiscoveredFiles>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { globFiles } from "./globFiles.js";
|
|
2
2
|
const CODE_PATTERNS = ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx", "**/*.mjs", "**/*.cjs"];
|
|
3
3
|
const PYTHON_PATTERNS = ["**/*.py"];
|
|
4
4
|
const HTML_PATTERNS = ["**/*.html", "**/*.htm"];
|
|
@@ -27,13 +27,13 @@ const PYTHON_IGNORE = [
|
|
|
27
27
|
"**/test_*.py",
|
|
28
28
|
"**/*_test.py",
|
|
29
29
|
];
|
|
30
|
-
export async function discoverFiles(root) {
|
|
30
|
+
export async function discoverFiles(root, boundary) {
|
|
31
31
|
const [codeFiles, pythonFiles, htmlFiles, schemaFiles, manifestFiles] = await Promise.all([
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
32
|
+
globFiles(CODE_PATTERNS, { cwd: root, absolute: true, ignore: IGNORE, dot: false, boundary }),
|
|
33
|
+
globFiles(PYTHON_PATTERNS, { cwd: root, absolute: true, ignore: PYTHON_IGNORE, dot: false, boundary }),
|
|
34
|
+
globFiles(HTML_PATTERNS, { cwd: root, absolute: true, ignore: IGNORE, dot: false, boundary }),
|
|
35
|
+
globFiles(SCHEMA_PATTERNS, { cwd: root, absolute: true, ignore: ["**/node_modules/**"], dot: false, boundary }),
|
|
36
|
+
globFiles(MANIFEST_PATTERNS, { cwd: root, absolute: true, ignore: ["**/node_modules/**"], dot: false, boundary }),
|
|
37
37
|
]);
|
|
38
38
|
return { codeFiles, pythonFiles, htmlFiles, schemaFiles, manifestFiles };
|
|
39
39
|
}
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
import type { RootBoundary } from "./rootBoundary.js";
|
|
1
2
|
import type { ProjectManifest } from "./types.js";
|
|
2
|
-
export
|
|
3
|
+
export interface ProjectDiscovery {
|
|
4
|
+
projects: ProjectManifest[];
|
|
5
|
+
/** Non-null when the scanned repo's own workspace config was rejected (malformed, hostile, or too
|
|
6
|
+
* expensive to expand) and the default package locations were scanned instead -- surfaced on the
|
|
7
|
+
* console and in the generated policy, since the scan may then be incomplete. */
|
|
8
|
+
workspaceWarning: string | null;
|
|
9
|
+
}
|
|
10
|
+
export declare function discoverProjects(root: string, boundary: RootBoundary): Promise<ProjectDiscovery>;
|
|
3
11
|
export declare function describeProject(root: string, project: ProjectManifest): string;
|
|
@@ -1,32 +1,85 @@
|
|
|
1
1
|
import { readFileSync, existsSync } from "node:fs";
|
|
2
2
|
import { dirname, join, relative } from "node:path";
|
|
3
|
-
import
|
|
4
|
-
|
|
3
|
+
import { globFiles } from "./globFiles.js";
|
|
4
|
+
import { validateWorkspacePatterns } from "./workspacePatterns.js";
|
|
5
|
+
const DEFAULT_WORKSPACE_PATTERNS = ["packages/*", "apps/*"];
|
|
6
|
+
function rejectedConfigWarning(reason) {
|
|
7
|
+
return `Workspace configuration in package.json was rejected (${reason}); the default package locations (${DEFAULT_WORKSPACE_PATTERNS.join(", ")}) were scanned instead, so this scan may be incomplete.`;
|
|
8
|
+
}
|
|
9
|
+
function readPackageJson(path, boundary) {
|
|
10
|
+
if (!boundary.allowsPath(path))
|
|
11
|
+
return { error: "package.json resolves outside the scanned root" };
|
|
12
|
+
try {
|
|
13
|
+
// A leading BOM (common from Windows editors) is valid in practice but not to JSON.parse.
|
|
14
|
+
const text = readFileSync(path, "utf8").replace(/^/, "");
|
|
15
|
+
return { value: JSON.parse(text) };
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return { error: "package.json could not be parsed" };
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
// undefined means "no workspace config at all"; anything else (including a wrong type) is handed to
|
|
22
|
+
// the validator, which rejects what isn't an array of well-formed patterns.
|
|
23
|
+
function rawWorkspaces(pkg) {
|
|
24
|
+
const workspaces = typeof pkg === "object" && pkg !== null ? pkg.workspaces : undefined;
|
|
25
|
+
if (workspaces === undefined || workspaces === null)
|
|
26
|
+
return undefined;
|
|
27
|
+
if (typeof workspaces === "object" && !Array.isArray(workspaces))
|
|
28
|
+
return workspaces.packages ?? undefined;
|
|
29
|
+
return workspaces;
|
|
30
|
+
}
|
|
31
|
+
async function globManifests(patterns, root, boundary) {
|
|
32
|
+
return globFiles(patterns.map((p) => (p.startsWith("!") ? `!${join(p.slice(1), "package.json")}` : join(p, "package.json"))), { cwd: root, absolute: true, ignore: ["**/node_modules/**"], boundary });
|
|
33
|
+
}
|
|
34
|
+
export async function discoverProjects(root, boundary) {
|
|
5
35
|
const rootPackageJsonPath = join(root, "package.json");
|
|
6
36
|
if (!existsSync(rootPackageJsonPath)) {
|
|
7
|
-
return [{ root, kind: "unknown", manifestPath: null }];
|
|
8
|
-
}
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
37
|
+
return { projects: [{ root, kind: "unknown", manifestPath: null }], workspaceWarning: null };
|
|
38
|
+
}
|
|
39
|
+
const rootProject = { root, kind: "node", manifestPath: rootPackageJsonPath };
|
|
40
|
+
const parsed = readPackageJson(rootPackageJsonPath, boundary);
|
|
41
|
+
let workspaceWarning = null;
|
|
42
|
+
let patterns = null;
|
|
43
|
+
let hasWorkspaceConfig = existsSync(join(root, "lerna.json")) || existsSync(join(root, "nx.json"));
|
|
44
|
+
if ("error" in parsed) {
|
|
45
|
+
workspaceWarning = rejectedConfigWarning(parsed.error);
|
|
46
|
+
hasWorkspaceConfig = true;
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
const raw = rawWorkspaces(parsed.value);
|
|
50
|
+
if (raw !== undefined) {
|
|
51
|
+
hasWorkspaceConfig = true;
|
|
52
|
+
const validation = validateWorkspacePatterns(raw);
|
|
53
|
+
if (validation.ok)
|
|
54
|
+
patterns = validation.patterns;
|
|
55
|
+
else
|
|
56
|
+
workspaceWarning = rejectedConfigWarning(validation.reason);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
16
59
|
if (!hasWorkspaceConfig) {
|
|
17
|
-
return
|
|
60
|
+
return { projects: [rootProject], workspaceWarning: null };
|
|
61
|
+
}
|
|
62
|
+
let manifestPaths;
|
|
63
|
+
try {
|
|
64
|
+
manifestPaths = await globManifests(patterns ?? DEFAULT_WORKSPACE_PATTERNS, root, boundary);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// Belt and braces behind validateWorkspacePatterns: a pattern that still fails to expand must
|
|
68
|
+
// never take the whole scan down.
|
|
69
|
+
workspaceWarning = rejectedConfigWarning("the patterns could not be expanded");
|
|
70
|
+
try {
|
|
71
|
+
manifestPaths = await globManifests(DEFAULT_WORKSPACE_PATTERNS, root, boundary);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
manifestPaths = [];
|
|
75
|
+
}
|
|
18
76
|
}
|
|
19
|
-
const patterns = workspacePatterns ?? ["packages/*", "apps/*"];
|
|
20
|
-
const manifestPaths = await fg(patterns.map((p) => join(p, "package.json")), { cwd: root, absolute: true, ignore: ["**/node_modules/**"] });
|
|
21
77
|
const projects = manifestPaths.map((manifestPath) => ({
|
|
22
78
|
root: dirname(manifestPath),
|
|
23
79
|
kind: "node",
|
|
24
80
|
manifestPath,
|
|
25
81
|
}));
|
|
26
|
-
|
|
27
|
-
return [{ root, kind: "node", manifestPath: rootPackageJsonPath }];
|
|
28
|
-
}
|
|
29
|
-
return projects;
|
|
82
|
+
return { projects: projects.length === 0 ? [rootProject] : projects, workspaceWarning };
|
|
30
83
|
}
|
|
31
84
|
export function describeProject(root, project) {
|
|
32
85
|
const rel = relative(root, project.root);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RootBoundary } from "./rootBoundary.js";
|
|
2
|
+
export interface GlobFilesOptions {
|
|
3
|
+
cwd: string;
|
|
4
|
+
absolute?: boolean;
|
|
5
|
+
ignore?: string[];
|
|
6
|
+
dot?: boolean;
|
|
7
|
+
/** When given, symlinks resolving outside the scanned root are neither walked nor returned. */
|
|
8
|
+
boundary?: RootBoundary;
|
|
9
|
+
}
|
|
10
|
+
export declare function globFiles(patterns: string | string[], options: GlobFilesOptions): Promise<string[]>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { glob } from "tinyglobby";
|
|
2
|
+
// expandDirectories: false keeps tinyglobby's matching identical to the fast-glob behavior this
|
|
3
|
+
// replaced (a pattern matches only what it names, never everything under a matched directory).
|
|
4
|
+
// tinyglobby/picomatch compiles braces to regex alternation rather than expanding them, so it has no
|
|
5
|
+
// equivalent of the `braces` package's stack-overflow/memory exposure (GHSA-vfj7-8cjw-p6xm); patterns
|
|
6
|
+
// that come from a scanned repo are still bounded first, in workspacePatterns.ts.
|
|
7
|
+
// Sorted so scan order (and therefore the order of findings in reports) doesn't depend on filesystem
|
|
8
|
+
// traversal order, which neither fast-glob nor tinyglobby guarantees.
|
|
9
|
+
export async function globFiles(patterns, options) {
|
|
10
|
+
const { boundary, ...globOptions } = options;
|
|
11
|
+
return (await glob(patterns, { expandDirectories: false, ...globOptions, ...(boundary ? { fs: boundary.globFs() } : {}) })).sort();
|
|
12
|
+
}
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
import type { RootBoundary } from "./rootBoundary.js";
|
|
2
|
+
export declare function detectUnsupportedStack(root: string, boundary: RootBoundary): Promise<string | null>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import
|
|
3
|
+
import { globFiles } from "./globFiles.js";
|
|
4
4
|
const IGNORE = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/.git/**", "**/vendor/**", "**/venv/**", "**/.venv/**", "**/__pycache__/**"];
|
|
5
5
|
// Python moved from UNSUPPORTED_LANGUAGES to here once Flask detection shipped (Stage A) -- it's a
|
|
6
6
|
// supported *language* now, but only one framework within it is actually understood. A Python project
|
|
@@ -17,7 +17,7 @@ const FASTAPI_IMPORT = /\bfrom\s+fastapi\b|\bimport\s+fastapi\b/;
|
|
|
17
17
|
function bannerize(message) {
|
|
18
18
|
return `UNSUPPORTED CODEBASE — ${message} SrcLawyer only analyzes JavaScript/TypeScript, HTML, and Flask-based Python today; the findings below (if any) are incomplete and MUST NOT be treated as a full compliance scan.`;
|
|
19
19
|
}
|
|
20
|
-
async function detectFrameworkSignature(root) {
|
|
20
|
+
async function detectFrameworkSignature(root, boundary) {
|
|
21
21
|
if (existsSync(join(root, "manage.py"))) {
|
|
22
22
|
return bannerize("Django detected (manage.py found at the project root) — Django support isn't built yet.");
|
|
23
23
|
}
|
|
@@ -26,7 +26,7 @@ async function detectFrameworkSignature(root) {
|
|
|
26
26
|
if (hasGemfile && hasRailsLayout) {
|
|
27
27
|
return bannerize("Rails detected (Gemfile + a Rails app/config layout found).");
|
|
28
28
|
}
|
|
29
|
-
const protoFiles = await
|
|
29
|
+
const protoFiles = await globFiles("**/*.proto", { cwd: root, ignore: IGNORE, dot: false, boundary });
|
|
30
30
|
if (protoFiles.length > 0) {
|
|
31
31
|
return bannerize(`gRPC/Protocol Buffers detected (${protoFiles.length} .proto file(s) found) — protobuf schemas are not parsed.`);
|
|
32
32
|
}
|
|
@@ -53,10 +53,10 @@ async function detectUnsupportedPythonFramework(root, pythonFiles) {
|
|
|
53
53
|
}
|
|
54
54
|
return null;
|
|
55
55
|
}
|
|
56
|
-
async function detectDominantUnsupportedLanguage(root) {
|
|
56
|
+
async function detectDominantUnsupportedLanguage(root, boundary) {
|
|
57
57
|
const [supportedFiles, ...unsupportedGroups] = await Promise.all([
|
|
58
|
-
|
|
59
|
-
...UNSUPPORTED_LANGUAGES.map((lang) =>
|
|
58
|
+
globFiles(SUPPORTED_EXTENSIONS.map((ext) => `**/*.${ext}`), { cwd: root, ignore: IGNORE, dot: false, boundary }),
|
|
59
|
+
...UNSUPPORTED_LANGUAGES.map((lang) => globFiles(lang.extensions.map((ext) => `**/*.${ext}`), { cwd: root, ignore: IGNORE, dot: false, boundary })),
|
|
60
60
|
]);
|
|
61
61
|
const counts = UNSUPPORTED_LANGUAGES.map((lang, i) => ({ label: lang.label, count: unsupportedGroups[i].length }));
|
|
62
62
|
const dominant = counts.filter((c) => c.count > 0).sort((a, b) => b.count - a.count)[0];
|
|
@@ -64,15 +64,15 @@ async function detectDominantUnsupportedLanguage(root) {
|
|
|
64
64
|
return null;
|
|
65
65
|
return bannerize(`${dominant.label} detected as the primary language (${dominant.count} file(s), vs ${supportedFiles.length} supported file(s)).`);
|
|
66
66
|
}
|
|
67
|
-
export async function detectUnsupportedStack(root) {
|
|
68
|
-
const frameworkWarning = await detectFrameworkSignature(root);
|
|
67
|
+
export async function detectUnsupportedStack(root, boundary) {
|
|
68
|
+
const frameworkWarning = await detectFrameworkSignature(root, boundary);
|
|
69
69
|
if (frameworkWarning)
|
|
70
70
|
return frameworkWarning;
|
|
71
|
-
const pythonFiles = await
|
|
71
|
+
const pythonFiles = await globFiles("**/*.py", { cwd: root, absolute: true, ignore: IGNORE, dot: false, boundary });
|
|
72
72
|
if (pythonFiles.length > 0) {
|
|
73
73
|
const pythonFrameworkWarning = await detectUnsupportedPythonFramework(root, pythonFiles);
|
|
74
74
|
if (pythonFrameworkWarning)
|
|
75
75
|
return pythonFrameworkWarning;
|
|
76
76
|
}
|
|
77
|
-
return detectDominantUnsupportedLanguage(root);
|
|
77
|
+
return detectDominantUnsupportedLanguage(root, boundary);
|
|
78
78
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { realpath, realpathSync } from "node:fs";
|
|
2
|
+
/**
|
|
3
|
+
* The scanned root as a security boundary: a scanned repo is untrusted input, so no file whose REAL
|
|
4
|
+
* path resolves outside it may be read, and no directory symlink leaving it may be walked (a symlink
|
|
5
|
+
* to "/" or "~" would otherwise make a scan traverse the whole filesystem and report lines from files
|
|
6
|
+
* the repo's author chose). In-root symlinks, including loops that stay inside, keep working.
|
|
7
|
+
*/
|
|
8
|
+
export declare class RootBoundary {
|
|
9
|
+
readonly root: string;
|
|
10
|
+
readonly realRoot: string;
|
|
11
|
+
private readonly blocked;
|
|
12
|
+
private readonly resolved;
|
|
13
|
+
constructor(root: string);
|
|
14
|
+
isInsideRealPath(resolved: string): boolean;
|
|
15
|
+
/** Records a blocked link by its in-repo (lexical) path only -- never its target. */
|
|
16
|
+
private recordBlocked;
|
|
17
|
+
/** Read-time check, independent of how the path was discovered: the real path if it is inside the
|
|
18
|
+
* root, else null (and an out-of-root link is recorded). Unresolvable paths fail closed. Cached. */
|
|
19
|
+
resolveInside(path: string): string | null;
|
|
20
|
+
allowsPath(path: string): boolean;
|
|
21
|
+
/** `fs` overrides for tinyglobby: a symlink resolving outside the root errors out, so it is neither
|
|
22
|
+
* walked (directories) nor returned (files). */
|
|
23
|
+
globFs(): {
|
|
24
|
+
realpath: typeof realpath;
|
|
25
|
+
realpathSync: typeof realpathSync;
|
|
26
|
+
};
|
|
27
|
+
/** Relative in-repo paths of every blocked link, sorted; call after discovery and reading are done. */
|
|
28
|
+
blockedLinks(): string[];
|
|
29
|
+
}
|