srclawyer 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +33 -2
  2. package/dist/cli.js +11 -1
  3. package/dist/cloud/protectedLogicClient.js +30 -6
  4. package/dist/commands/init.d.ts +5 -0
  5. package/dist/commands/init.js +36 -11
  6. package/dist/commands/scan.js +9 -3
  7. package/dist/engine/discoverFiles.d.ts +2 -1
  8. package/dist/engine/discoverFiles.js +7 -7
  9. package/dist/engine/discoverProjects.d.ts +9 -1
  10. package/dist/engine/discoverProjects.js +71 -18
  11. package/dist/engine/globFiles.d.ts +10 -0
  12. package/dist/engine/globFiles.js +12 -0
  13. package/dist/engine/languageDetection.d.ts +2 -1
  14. package/dist/engine/languageDetection.js +10 -10
  15. package/dist/engine/rootBoundary.d.ts +29 -0
  16. package/dist/engine/rootBoundary.js +102 -0
  17. package/dist/engine/scanEngine.js +174 -58
  18. package/dist/engine/scanLimits.d.ts +29 -0
  19. package/dist/engine/scanLimits.js +70 -0
  20. package/dist/engine/tsProject.d.ts +2 -1
  21. package/dist/engine/tsProject.js +40 -3
  22. package/dist/engine/types.d.ts +16 -0
  23. package/dist/engine/workspacePatterns.d.ts +23 -0
  24. package/dist/engine/workspacePatterns.js +140 -0
  25. package/dist/llm/buildRequestItem.d.ts +5 -1
  26. package/dist/llm/buildRequestItem.js +6 -2
  27. package/dist/llm/resolveAmbiguousFindings.js +7 -3
  28. package/dist/llm/resolveProviderForTier.js +1 -1
  29. package/dist/policy/generatePolicy.js +33 -8
  30. package/dist/policy/legalNotices.js +11 -7
  31. package/dist/report/formatText.d.ts +6 -1
  32. package/dist/report/formatText.js +23 -2
  33. package/dist/rules/htmlRules.d.ts +11 -0
  34. package/dist/rules/htmlRules.js +73 -17
  35. package/dist/rules/zodPreFilter.js +10 -1
  36. package/dist/schemaParsers/openapi.d.ts +6 -1
  37. package/dist/schemaParsers/openapi.js +3 -1
  38. package/package.json +2 -2
@@ -0,0 +1,140 @@
1
+ /**
2
+ * Validation for workspace glob patterns read from a SCANNED repo's own package.json -- i.e. input an
3
+ * attacker controls whenever the repo being scanned isn't the user's own. Brace expansion in glob
4
+ * libraries can be driven to stack overflow, memory exhaustion, or minutes of CPU by very short
5
+ * patterns (a 37-character `{1..300}{1..300}{1..300}` is enough), so patterns are bounded here before
6
+ * they ever reach a glob library. Real workspace patterns (`packages/*`, `{apps,packages}/*`,
7
+ * `tools/*\/packages/*`) are tens of characters with at most one small brace group.
8
+ */
9
+ export const MAX_WORKSPACE_PATTERNS = 100;
10
+ export const MAX_PATTERN_LENGTH = 200;
11
+ export const MAX_BRACE_DEPTH = 2;
12
+ export const MAX_BRACE_GROUPS = 3;
13
+ export const MAX_BRACE_EXPANSION = 64;
14
+ export const MAX_RANGE_SPAN = 100;
15
+ const NUMERIC_RANGE = /^(-?\d+)\.\.(-?\d+)(?:\.\.(-?\d+))?$/;
16
+ const ALPHA_RANGE = /^[A-Za-z]\.\.[A-Za-z](?:\.\.-?\d+)?$/;
17
+ /** Upper bound on how many strings a brace-bearing segment expands to; null if malformed or too wide. */
18
+ function estimateExpansion(segment) {
19
+ let total = 1;
20
+ let i = 0;
21
+ while (i < segment.length) {
22
+ if (segment[i] === "}")
23
+ return null;
24
+ if (segment[i] !== "{") {
25
+ i += 1;
26
+ continue;
27
+ }
28
+ let depth = 1;
29
+ let j = i + 1;
30
+ while (j < segment.length && depth > 0) {
31
+ if (segment[j] === "{")
32
+ depth += 1;
33
+ else if (segment[j] === "}")
34
+ depth -= 1;
35
+ j += 1;
36
+ }
37
+ if (depth !== 0)
38
+ return null;
39
+ const inner = segment.slice(i + 1, j - 1);
40
+ const groupEstimate = estimateGroup(inner);
41
+ if (groupEstimate === null)
42
+ return null;
43
+ total *= groupEstimate;
44
+ if (total > MAX_BRACE_EXPANSION)
45
+ return null;
46
+ i = j;
47
+ }
48
+ return total;
49
+ }
50
+ function estimateGroup(inner) {
51
+ const numeric = NUMERIC_RANGE.exec(inner);
52
+ if (numeric) {
53
+ const span = Math.abs(Number(numeric[2]) - Number(numeric[1])) + 1;
54
+ const step = Math.abs(Number(numeric[3] ?? 1)) || 1;
55
+ return span > MAX_RANGE_SPAN ? null : Math.ceil(span / step);
56
+ }
57
+ if (ALPHA_RANGE.test(inner))
58
+ return 52;
59
+ const parts = [];
60
+ let depth = 0;
61
+ let start = 0;
62
+ for (let k = 0; k < inner.length; k++) {
63
+ if (inner[k] === "{")
64
+ depth += 1;
65
+ else if (inner[k] === "}")
66
+ depth -= 1;
67
+ else if (inner[k] === "," && depth === 0) {
68
+ parts.push(inner.slice(start, k));
69
+ start = k + 1;
70
+ }
71
+ }
72
+ parts.push(inner.slice(start));
73
+ let sum = 0;
74
+ for (const part of parts) {
75
+ const estimate = estimateExpansion(part);
76
+ if (estimate === null)
77
+ return null;
78
+ sum += estimate;
79
+ }
80
+ return sum;
81
+ }
82
+ function braceShapeProblem(pattern) {
83
+ let depth = 0;
84
+ let maxDepth = 0;
85
+ let groups = 0;
86
+ for (const ch of pattern) {
87
+ if (ch === "{") {
88
+ depth += 1;
89
+ groups += 1;
90
+ maxDepth = Math.max(maxDepth, depth);
91
+ }
92
+ else if (ch === "}") {
93
+ depth -= 1;
94
+ if (depth < 0)
95
+ return "unbalanced braces";
96
+ }
97
+ }
98
+ if (depth !== 0)
99
+ return "unbalanced braces";
100
+ if (maxDepth > MAX_BRACE_DEPTH)
101
+ return `brace nesting deeper than ${MAX_BRACE_DEPTH}`;
102
+ if (groups > MAX_BRACE_GROUPS)
103
+ return `more than ${MAX_BRACE_GROUPS} brace groups`;
104
+ if (estimateExpansion(pattern) === null)
105
+ return `brace expansion wider than ${MAX_BRACE_EXPANSION} alternatives`;
106
+ return null;
107
+ }
108
+ function patternProblem(pattern) {
109
+ if (typeof pattern !== "string")
110
+ return "an entry is not a string";
111
+ if (pattern.length === 0)
112
+ return "an entry is empty";
113
+ if (pattern.length > MAX_PATTERN_LENGTH)
114
+ return `an entry is longer than ${MAX_PATTERN_LENGTH} characters`;
115
+ if (pattern.includes("\0"))
116
+ return "an entry contains a NUL character";
117
+ if (pattern.includes("\\"))
118
+ return "an entry contains a backslash";
119
+ const body = pattern.startsWith("!") ? pattern.slice(1) : pattern;
120
+ if (body.startsWith("!"))
121
+ return "an entry has more than one leading '!'";
122
+ if (body.startsWith("/") || /^[A-Za-z]:/.test(body))
123
+ return "an entry is an absolute path";
124
+ if (body.split("/").includes(".."))
125
+ return "an entry escapes the repo root with '..'";
126
+ return braceShapeProblem(body);
127
+ }
128
+ /** `raw` is whatever package.json's `workspaces` (or `workspaces.packages`) held, unvalidated. */
129
+ export function validateWorkspacePatterns(raw) {
130
+ if (!Array.isArray(raw))
131
+ return { ok: false, reason: "workspaces is not an array of patterns" };
132
+ if (raw.length > MAX_WORKSPACE_PATTERNS)
133
+ return { ok: false, reason: `more than ${MAX_WORKSPACE_PATTERNS} workspace patterns` };
134
+ for (const pattern of raw) {
135
+ const problem = patternProblem(pattern);
136
+ if (problem)
137
+ return { ok: false, reason: problem };
138
+ }
139
+ return { ok: true, patterns: raw };
140
+ }
@@ -6,5 +6,9 @@ import type { LlmRequestItem } from "./types.js";
6
6
  * they leave this function — LlmRequestItem has no raw-evidence field, so a
7
7
  * provider implementation is structurally incapable of receiving
8
8
  * unredacted content.
9
+ *
10
+ * `requestId` is an opaque per-scan identifier (f0, f1, ...), never finding.id: finding ids embed the
11
+ * file path, line number and field name, and none of those may reach the provider. The caller
12
+ * (resolveAmbiguousFindings) owns the mapping back to findings and never sends it anywhere.
9
13
  */
10
- export declare function buildRequestItem(finding: Finding, collectsChildrensData: boolean): LlmRequestItem;
14
+ export declare function buildRequestItem(finding: Finding, collectsChildrensData: boolean, requestId: string): LlmRequestItem;
@@ -5,10 +5,14 @@ import { redactSecrets } from "../engine/secretRedaction.js";
5
5
  * they leave this function — LlmRequestItem has no raw-evidence field, so a
6
6
  * provider implementation is structurally incapable of receiving
7
7
  * unredacted content.
8
+ *
9
+ * `requestId` is an opaque per-scan identifier (f0, f1, ...), never finding.id: finding ids embed the
10
+ * file path, line number and field name, and none of those may reach the provider. The caller
11
+ * (resolveAmbiguousFindings) owns the mapping back to findings and never sends it anywhere.
8
12
  */
9
- export function buildRequestItem(finding, collectsChildrensData) {
13
+ export function buildRequestItem(finding, collectsChildrensData, requestId) {
10
14
  return {
11
- findingId: finding.id,
15
+ findingId: requestId,
12
16
  redactedEvidence: redactSecrets(finding.evidence),
13
17
  redactedDescription: redactSecrets(finding.description),
14
18
  dataCategories: finding.dataCategories,
@@ -6,8 +6,11 @@ const DEFAULT_BATCH_CHAR_BUDGET = 6000;
6
6
  export async function resolveAmbiguousFindings(findings, provider, options) {
7
7
  const batches = chunkFindings(findings, options.batchSize ?? DEFAULT_BATCH_SIZE);
8
8
  const resolvedById = new Map();
9
+ // Opaque per-scan ids (f0, f1, ...), unique across all batches. Finding ids embed file paths and
10
+ // line numbers, so only these ever go on the wire; the mapping back lives here and nowhere else.
11
+ const requestIdOf = new Map(findings.map((finding, i) => [finding, `f${i}`]));
9
12
  for (const batch of batches) {
10
- const items = batch.map((finding) => buildRequestItem(finding, options.collectsChildrensData));
13
+ const items = batch.map((finding) => buildRequestItem(finding, options.collectsChildrensData, requestIdOf.get(finding)));
11
14
  const isHighStakesById = new Map(items.map((item) => [item.findingId, item.isHighStakes]));
12
15
  let responses;
13
16
  try {
@@ -27,8 +30,9 @@ export async function resolveAmbiguousFindings(findings, provider, options) {
27
30
  }
28
31
  const responseById = new Map(responses.map((response) => [response.findingId, response]));
29
32
  for (const finding of batch) {
30
- const response = responseById.get(finding.id);
31
- const isHighStakes = isHighStakesById.get(finding.id) ?? false;
33
+ const requestId = requestIdOf.get(finding);
34
+ const response = responseById.get(requestId);
35
+ const isHighStakes = isHighStakesById.get(requestId) ?? false;
32
36
  resolvedById.set(finding.id, toResolution(response, isHighStakes, provider));
33
37
  }
34
38
  }
@@ -10,7 +10,7 @@ export function resolveProviderForTier(config) {
10
10
  if (!envVar) {
11
11
  return {
12
12
  provider: null,
13
- error: `Tier is "byok" but no API key environment variable is configured in ${CONFIG_FILENAME}. Run "srclawyer init" to set one.`,
13
+ error: `Tier is "byok" but no API key environment variable is configured in ${CONFIG_FILENAME}. Add "llm: { provider: anthropic, apiKeyEnvVar: <ENV_VAR_NAME> }" to it, or delete the file and re-run "srclawyer init --enable-ai-review".`,
14
14
  };
15
15
  }
16
16
  const apiKey = process.env[envVar];
@@ -108,6 +108,17 @@ function renderUnsupportedStackWarning(warning) {
108
108
  function renderProtectedLogicWarning(warning) {
109
109
  return ["> **WARNING: PROTECTED-LOGIC SERVICE UNAVAILABLE — POLICY MAY BE INCOMPLETE**", ">", `> ${warning}`].join("\n");
110
110
  }
111
+ // Same reasoning as the other warning banners: the policy file is committed and shared without the
112
+ // console output, and a rejected workspace config means packages outside the default locations were
113
+ // never scanned.
114
+ function renderWorkspaceWarning(warning) {
115
+ return ["> **WARNING: WORKSPACE CONFIGURATION REJECTED — SCAN MAY BE INCOMPLETE**", ">", `> ${warning}`].join("\n");
116
+ }
117
+ // Same reasoning as the other banners: files skipped for size mean findings in them are missing from
118
+ // a policy file that will be committed and shared without the console output.
119
+ function renderSkippedFilesWarning(warning) {
120
+ return ["> **WARNING: FILES SKIPPED OR NOT ANALYZED — SCAN INCOMPLETE**", ">", `> ${warning}`].join("\n");
121
+ }
111
122
  function placeholder(label) {
112
123
  return `[TO COMPLETE: ${label}]`;
113
124
  }
@@ -154,16 +165,22 @@ function renderIdentitySection(config, placeholders) {
154
165
  // can simultaneously list detected third-party integrations (e.g. Stripe) in the Third-Party Data
155
166
  // Processors section above. These sentences deliberately (1) use CCPA/CPRA's own defined terms rather
156
167
  // than casual language, (2) frame the statement as the user's own self-declared setup answer, not a
157
- // legal conclusion drawn by this tool, and (3) always close with an explicit cross-reference back to
158
- // the detected-processors section, in all three states (true/false/placeholder), so the two sections
159
- // are never read in isolation from each other. See docs/legal-wording-review.md -- this is drafted,
168
+ // legal conclusion drawn by this tool, and (3) close with a cross-reference back to the
169
+ // detected-processors section, in all three states (true/false/placeholder), so the two sections are
170
+ // never read in isolation from each other -- EXCEPT that the cross-reference only names that section
171
+ // when it actually exists in this document (hasProcessorSection); otherwise it would point at a
172
+ // heading that was never rendered, whether because nothing was detected or because detection itself
173
+ // was incomplete (unsupported stack). See docs/legal-wording-review.md -- this is drafted,
160
174
  // attorney-reviewable starting wording, not settled language.
161
- function renderDataPracticesSection(config, placeholders) {
175
+ function renderDataPracticesSection(config, placeholders, hasProcessorSection) {
162
176
  const retention = valueOrPlaceholder(config?.dataRetentionPolicy, "data retention period or policy", placeholders);
163
177
  const deletion = valueOrPlaceholder(config?.accountDeletionInstructions, "how a user can delete their account or request data deletion", placeholders);
164
178
  const sellShare = triStateSentence(config?.sellsOrSharesData, 'whether personal data is sold or shared with third parties, as "sell" and "share" are defined under applicable law (e.g. CCPA/CPRA)', 'Based on information provided during setup, this product has been declared to sell or share personal data with third parties, as "sell" and "share" are defined under applicable law (e.g. CCPA/CPRA).', 'Based on information provided during setup, this product has not been declared to sell or share personal data with third parties, as "sell" and "share" are defined under applicable law (e.g. CCPA/CPRA).', placeholders);
165
179
  const advertising = triStateSentence(config?.usesDataForAdvertising, "whether personal data is used for cross-context behavioral advertising", "Based on information provided during setup, this product has been declared to use personal data for cross-context behavioral advertising.", "Based on information provided during setup, this product has not been declared to use personal data for cross-context behavioral advertising.", placeholders);
166
180
  const aiServices = triStateSentence(config?.sendsDataToAiServices, "whether user data is sent to third-party AI services as part of this product's own functionality", "User data may be sent to third-party AI services as part of this product's own functionality.", "User data is not sent to third-party AI services as part of this product's own functionality.", placeholders);
181
+ const processorsCrossReference = hasProcessorSection
182
+ ? "See the Third-Party Data Processors section above for the specific third-party integrations detected in this codebase; the declarations above describe this product's own stated practices and do not override what was actually detected there."
183
+ : "This document does not include a Third-Party Data Processors section (none were detected, or detection may have been incomplete -- see any warning above); the declarations above describe this product's own stated practices only and do not override any third-party integrations this codebase may actually use.";
167
184
  return [
168
185
  "## Data Retention, Deletion, and Use",
169
186
  "",
@@ -171,7 +188,7 @@ function renderDataPracticesSection(config, placeholders) {
171
188
  `To delete your account or request deletion of your personal data: ${deletion}`,
172
189
  sellShare,
173
190
  advertising,
174
- "See the Third-Party Data Processors section above for the specific third-party integrations detected in this codebase; the declarations above describe this product's own stated practices and do not override what was actually detected there.",
191
+ processorsCrossReference,
175
192
  aiServices,
176
193
  ].join("\n");
177
194
  }
@@ -228,17 +245,26 @@ export function generatePolicyMarkdown(result, config, llmError, protectedLogicW
228
245
  });
229
246
  const declinedFindings = result.findings.filter((f) => outcomeOf(f) === "declined");
230
247
  const reviewFindings = result.findings.filter((f) => outcomeOf(f) === "unresolved");
231
- // Rendered first (before the final parts[] is assembled) so the shared placeholders array is fully
248
+ // Computed first so renderDataPracticesSection knows whether the Third-Party Data Processors
249
+ // section (rendered later, below) actually exists in this document before it cross-references it.
250
+ const processorSection = renderProcessorSection(confidentFindings);
251
+ // Rendered next (before the final parts[] is assembled) so the shared placeholders array is fully
232
252
  // populated in time for the top-of-document summary below.
233
253
  const placeholders = [];
234
254
  const identitySection = renderIdentitySection(config, placeholders);
235
- const dataPracticesSection = renderDataPracticesSection(config, placeholders);
255
+ const dataPracticesSection = renderDataPracticesSection(config, placeholders, processorSection !== null);
236
256
  const rightsSection = renderRightsSection(config, placeholders);
237
257
  const childrensPrivacySection = renderChildrensPrivacySection(config, placeholders);
238
258
  const parts = ["# Privacy Policy", ""];
239
259
  if (result.unsupportedStackWarning) {
240
260
  parts.push(renderUnsupportedStackWarning(result.unsupportedStackWarning), "");
241
261
  }
262
+ if (result.workspaceWarning) {
263
+ parts.push(renderWorkspaceWarning(result.workspaceWarning), "");
264
+ }
265
+ if (result.skippedFilesWarning) {
266
+ parts.push(renderSkippedFilesWarning(result.skippedFilesWarning), "");
267
+ }
242
268
  if (protectedLogicWarning) {
243
269
  parts.push(renderProtectedLogicWarning(protectedLogicWarning), "");
244
270
  }
@@ -259,7 +285,6 @@ export function generatePolicyMarkdown(result, config, llmError, protectedLogicW
259
285
  else {
260
286
  parts.push("No data-collection patterns were confidently identified in this codebase.");
261
287
  }
262
- const processorSection = renderProcessorSection(confidentFindings);
263
288
  if (processorSection) {
264
289
  parts.push("", processorSection);
265
290
  }
@@ -8,16 +8,20 @@
8
8
  // notice and disclaimer, not a substitute for either.
9
9
  export const PRIVACY_POLICY_URL = "https://github.com/SrcLawyer/srclawyer#legal";
10
10
  export const TERMS_URL = "https://github.com/SrcLawyer/srclawyer#legal";
11
- // Verbatim from README.md's own "Network calls" section (minus markdown backticks, since this prints
12
- // to a terminal) -- one wording for this claim, not a second paraphrase that can drift from the first.
11
+ // Mirrors README.md's own "Network calls" section (condensed, minus markdown backticks since this
12
+ // prints to a terminal) -- the same claims in the same terms, not a second paraphrase that can drift.
13
+ // What is actually on the wire is pinned by tests/requestBodiesNoPaths.test.ts.
13
14
  export const NETWORK_CALLS_NOTICE = [
14
15
  "Heads up: SrcLawyer makes network calls.",
15
16
  "",
16
17
  "Detection itself is fully local and offline — no code leaves your machine for pattern matching.",
17
- "Once detection completes, scan makes one batched network call to grade the confidence of what it",
18
- "found (field names and derived yes/no signals only, never source code), and, for scans that use Zod",
19
- "schemas as request validators, one additional call sending a small redacted code fragment. This runs",
20
- "on every scan regardless of tier.",
18
+ "Once detection completes, every scan makes one batched call to grade the confidence of what it",
19
+ "found. For each field it sends an opaque id (f0, f1, ...), the data category the field's name maps",
20
+ "to, and three derived signals (is the name ambiguous, was it read directly from the request, how",
21
+ "many similar fields sit alongside it) -- no field names, file paths, line numbers or source code.",
22
+ "Scans that use Zod schemas as request validators make one additional batched call, sending an",
23
+ "opaque id and a small secret-redacted code fragment per schema (the schema, its validation call",
24
+ "and the imports it uses) -- never a file path or line number.",
21
25
  "",
22
26
  `Legal notice and disclaimer: ${PRIVACY_POLICY_URL}`,
23
27
  "(A dedicated privacy policy and terms of service are coming; this points to what exists today.)",
@@ -25,5 +29,5 @@ export const NETWORK_CALLS_NOTICE = [
25
29
  export const SCAN_DISCLAIMER_REMINDER = "Reminder: this is automated analysis, not legal advice — review the generated policy before publishing it.";
26
30
  export function byokNetworkNotice(apiKeyEnvVar) {
27
31
  const keySource = apiKeyEnvVar ? `env var ${apiKeyEnvVar}` : "your configured key";
28
- return `BYOK is configured: redacted code snippets for ambiguous findings will be sent to Anthropic under your own API key (${keySource}), for AI-assisted classification only.`;
32
+ return `BYOK is configured: for each finding scan couldn't classify, it will send Anthropic, under your own API key (${keySource}), an opaque id, the finding's data category and description (which names the field), and the one source line it was found on with secrets redacted — not the file's path or line number, and no other code — for AI-assisted classification only.`;
29
33
  }
@@ -1,2 +1,7 @@
1
1
  import type { ScanResult } from "../engine/types.js";
2
- export declare function formatText(result: ScanResult): string;
2
+ export interface FormatTextOptions {
3
+ /** True when the config has AI-assisted review (tier: byok) turned on -- the hint about that
4
+ * advanced option is only relevant when it isn't. */
5
+ aiReviewEnabled?: boolean;
6
+ }
7
+ export declare function formatText(result: ScanResult, options?: FormatTextOptions): string;
@@ -1,5 +1,5 @@
1
1
  const BANNER_RULE = "=".repeat(72);
2
- export function formatText(result) {
2
+ export function formatText(result, options = {}) {
3
3
  const lines = [];
4
4
  if (result.unsupportedStackWarning) {
5
5
  lines.push(BANNER_RULE);
@@ -8,7 +8,23 @@ export function formatText(result) {
8
8
  lines.push(BANNER_RULE);
9
9
  lines.push("");
10
10
  }
11
+ if (result.workspaceWarning) {
12
+ lines.push(BANNER_RULE);
13
+ lines.push("WARNING:");
14
+ lines.push(result.workspaceWarning);
15
+ lines.push(BANNER_RULE);
16
+ lines.push("");
17
+ }
18
+ if (result.skippedFilesWarning) {
19
+ lines.push(BANNER_RULE);
20
+ lines.push("WARNING:");
21
+ lines.push(result.skippedFilesWarning);
22
+ lines.push(BANNER_RULE);
23
+ lines.push("");
24
+ }
11
25
  lines.push(`Scanned ${result.filesScanned} file(s) across ${result.projects.length} project(s).`);
26
+ if (result.templateSyntaxNote)
27
+ lines.push(`Note: ${result.templateSyntaxNote}`);
12
28
  lines.push(`Found ${result.findings.length} finding(s), ${result.ambiguousCount} requiring review.`);
13
29
  lines.push("");
14
30
  const byProcessor = result.findings.filter((f) => f.processor);
@@ -38,7 +54,12 @@ export function formatText(result) {
38
54
  }
39
55
  else if (result.ambiguousCount > 0) {
40
56
  lines.push("");
41
- lines.push(`${result.ambiguousCount} item(s) require deeper analysis before they can be included in a policy. Run "srclawyer scan" with the "byok" tier configured (bring your own Anthropic key) to attempt AI-assisted resolution, or resolve manually. A "managed" tier is planned but not yet available.`);
57
+ const base = `${result.ambiguousCount} item(s) require deeper analysis before they can be included in a policy; review them manually.`;
58
+ // Only worth mentioning when the advanced option isn't already on, and only as a pointer -- it is
59
+ // opt-in (init --enable-ai-review, or a hand-edited config), not something to recommend to everyone.
60
+ lines.push(options.aiReviewEnabled
61
+ ? base
62
+ : `${base} (Optional, advanced: AI-assisted resolution with your own Anthropic API key — see "AI-assisted review" in the README.)`);
42
63
  }
43
64
  return lines.join("\n");
44
65
  }
@@ -1,6 +1,17 @@
1
1
  import type { Finding } from "../engine/types.js";
2
+ export declare const MAX_INLINE_SCRIPT_PADDING_CHARS = 16000000;
3
+ export declare function findInputTags(html: string): Array<{
4
+ index: number;
5
+ tag: string;
6
+ }>;
2
7
  export declare function runHtmlInputRules(html: string, filePath: string, root: string): Finding[];
3
8
  export interface ExtractedScript {
4
9
  code: string;
5
10
  }
11
+ export interface InlineScriptExtraction {
12
+ scripts: ExtractedScript[];
13
+ /** Inline scripts not extracted because the padding budget ran out (see MAX_INLINE_SCRIPT_PADDING_CHARS). */
14
+ skippedCount: number;
15
+ }
16
+ export declare function extractInlineScriptsBounded(html: string): InlineScriptExtraction;
6
17
  export declare function extractInlineScripts(html: string): ExtractedScript[];
@@ -1,27 +1,65 @@
1
1
  import { relative } from "node:path";
2
2
  import { classifyFieldName } from "./dataCategoryLexicon.js";
3
3
  import { isSafeFieldName } from "./safeFieldNames.js";
4
- const INPUT_TAG_RE = /<input\b[^>]*>/gi;
5
- const SCRIPT_TAG_RE = /<script([^>]*)>([\s\S]*?)<\/script>/gi;
4
+ // Tag matching is a start-anchor regex plus indexOf, not `<input\b[^>]*>` / `<script([^>]*)>([\s\S]*?)</script>`:
5
+ // on a file full of unclosed tags those single regexes rescan to the end of the file from every start
6
+ // (quadratic -- 200KB of "<input " took 8s), whereas this finds the same matches in one linear pass,
7
+ // since a tag with no closing delimiter after one start can't have one after any later start either.
8
+ const INPUT_START_RE = /<input\b/gi;
9
+ const SCRIPT_START_RE = /<script/gi;
10
+ const SCRIPT_CLOSE_RE = /<\/script>/gi;
11
+ // Padding inline scripts with one "\n" per preceding line (so rule line numbers match the original
12
+ // file) costs O(script count x line count) characters on a file with many scripts; this bounds the
13
+ // total so a hostile file can't exhaust memory. Scripts past the budget are reported, never dropped
14
+ // silently.
15
+ export const MAX_INLINE_SCRIPT_PADDING_CHARS = 16_000_000;
6
16
  const NON_DATA_INPUT_TYPES = new Set(["submit", "button", "reset", "image"]);
7
17
  function getAttr(tag, name) {
8
18
  const match = new RegExp(`${name}\\s*=\\s*["']([^"']*)["']`, "i").exec(tag);
9
19
  return match ? match[1] : null;
10
20
  }
11
- function lineAt(html, index) {
12
- return html.slice(0, index).split("\n").length;
21
+ // 1-based line number of a character offset, via a one-time index of line starts and a binary search
22
+ // (the previous slice-and-split per lookup was O(file size) each).
23
+ function makeLineLookup(text) {
24
+ const lineStarts = [0];
25
+ for (let i = text.indexOf("\n"); i !== -1; i = text.indexOf("\n", i + 1))
26
+ lineStarts.push(i + 1);
27
+ return (index) => {
28
+ let lo = 0;
29
+ let hi = lineStarts.length - 1;
30
+ while (lo < hi) {
31
+ const mid = (lo + hi + 1) >> 1;
32
+ if (lineStarts[mid] <= index)
33
+ lo = mid;
34
+ else
35
+ hi = mid - 1;
36
+ }
37
+ return lo + 1;
38
+ };
39
+ }
40
+ export function findInputTags(html) {
41
+ const tags = [];
42
+ const startRe = new RegExp(INPUT_START_RE.source, INPUT_START_RE.flags);
43
+ let match;
44
+ while ((match = startRe.exec(html))) {
45
+ const end = html.indexOf(">", match.index);
46
+ if (end === -1)
47
+ break;
48
+ startRe.lastIndex = end + 1;
49
+ tags.push({ index: match.index, tag: html.slice(match.index, end + 1) });
50
+ }
51
+ return tags;
13
52
  }
14
53
  export function runHtmlInputRules(html, filePath, root) {
15
54
  const relPath = relative(root, filePath);
16
55
  const findings = [];
17
- let match;
18
- while ((match = INPUT_TAG_RE.exec(html))) {
19
- const tag = match[0];
56
+ const lineAt = makeLineLookup(html);
57
+ for (const { index, tag } of findInputTags(html)) {
20
58
  const type = (getAttr(tag, "type") ?? "text").toLowerCase();
21
59
  if (NON_DATA_INPUT_TYPES.has(type))
22
60
  continue;
23
61
  const identifier = getAttr(tag, "id") ?? getAttr(tag, "name");
24
- const line = lineAt(html, match.index);
62
+ const line = lineAt(index);
25
63
  if (type === "password") {
26
64
  findings.push(makeFinding(relPath, line, tag, "authentication_credentials", "high", false, `<input type="password"> field collects a credential.`));
27
65
  continue;
@@ -56,20 +94,38 @@ function makeFinding(relPath, line, tag, category, confidence, requiresReview, d
56
94
  requiresReview,
57
95
  };
58
96
  }
59
- export function extractInlineScripts(html) {
97
+ export function extractInlineScriptsBounded(html) {
60
98
  const scripts = [];
99
+ const lineAt = makeLineLookup(html);
100
+ const startRe = new RegExp(SCRIPT_START_RE.source, SCRIPT_START_RE.flags);
101
+ const closeRe = new RegExp(SCRIPT_CLOSE_RE.source, SCRIPT_CLOSE_RE.flags);
102
+ let paddingUsed = 0;
103
+ let skippedCount = 0;
61
104
  let match;
62
- while ((match = SCRIPT_TAG_RE.exec(html))) {
63
- const attrs = match[1];
105
+ while ((match = startRe.exec(html))) {
106
+ const openEnd = html.indexOf(">", match.index);
107
+ if (openEnd === -1)
108
+ break;
109
+ closeRe.lastIndex = openEnd + 1;
110
+ const close = closeRe.exec(html);
111
+ if (!close)
112
+ break;
113
+ startRe.lastIndex = close.index + close[0].length;
114
+ const attrs = html.slice(match.index + "<script".length, openEnd);
64
115
  if (/\bsrc\s*=/i.test(attrs))
65
116
  continue;
66
117
  if (/\btype\s*=\s*["'](?!(?:text\/javascript|module|application\/javascript)["'])[^"']*["']/i.test(attrs))
67
118
  continue;
68
- const body = match[2];
69
- const openTagLength = "<script".length + attrs.length + 1;
70
- const contentStart = match.index + openTagLength;
71
- const linesBefore = html.slice(0, contentStart).split("\n").length - 1;
72
- scripts.push({ code: "\n".repeat(linesBefore) + body });
119
+ const linesBefore = lineAt(openEnd + 1) - 1;
120
+ if (paddingUsed + linesBefore > MAX_INLINE_SCRIPT_PADDING_CHARS) {
121
+ skippedCount += 1;
122
+ continue;
123
+ }
124
+ paddingUsed += linesBefore;
125
+ scripts.push({ code: "\n".repeat(linesBefore) + html.slice(openEnd + 1, close.index) });
73
126
  }
74
- return scripts;
127
+ return { scripts, skippedCount };
128
+ }
129
+ export function extractInlineScripts(html) {
130
+ return extractInlineScriptsBounded(html).scripts;
75
131
  }
@@ -101,6 +101,15 @@ function findElidableInnerLines(file, rootRange, protectedLines) {
101
101
  });
102
102
  return elidable;
103
103
  }
104
+ function escapeRegExp(text) {
105
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
106
+ }
107
+ // Whole-identifier match with explicit identifier-character boundaries, not `\b${name}\b`: `$` is a
108
+ // legal identifier character but a regex anchor, and `\b` doesn't treat it as a word character, so the
109
+ // old form silently never matched an import named like `$schema` and dropped it from the fragment.
110
+ function mentionsIdentifier(text, name) {
111
+ return new RegExp(`(?<![\\w$])${escapeRegExp(name)}(?![\\w$])`).test(text);
112
+ }
104
113
  function importedLocalNames(decl) {
105
114
  return decl.specifiers.map((spec) => spec.local.name);
106
115
  }
@@ -166,7 +175,7 @@ export function findZodCandidates(source, filePath, root) {
166
175
  for (const stmt of body) {
167
176
  if (stmt.type !== "ImportDeclaration")
168
177
  continue;
169
- const referenced = importedLocalNames(stmt).some((name) => new RegExp(`\\b${name}\\b`).test(coreText));
178
+ const referenced = importedLocalNames(stmt).some((name) => mentionsIdentifier(coreText, name));
170
179
  if (referenced)
171
180
  includeRange(lineRange(stmt));
172
181
  }
@@ -1,2 +1,7 @@
1
1
  import type { Finding } from "../engine/types.js";
2
- export declare function parseOpenApiSpec(filePath: string, root: string): Finding[];
2
+ /** Set by parseOpenApiSpec so the caller can surface a malformed document instead of silently getting
3
+ * fewer (or no) findings -- the YAML parser recovers from errors rather than throwing. */
4
+ export interface OpenApiParseReport {
5
+ hadErrors: boolean;
6
+ }
7
+ export declare function parseOpenApiSpec(filePath: string, root: string, report?: OpenApiParseReport): Finding[];
@@ -3,10 +3,12 @@ import { relative } from "node:path";
3
3
  import { parseDocument, LineCounter, isMap, isPair, isScalar } from "yaml";
4
4
  import { classifyFieldName, isAmbiguousFieldName } from "../rules/dataCategoryLexicon.js";
5
5
  import { isSafeFieldName } from "../rules/safeFieldNames.js";
6
- export function parseOpenApiSpec(filePath, root) {
6
+ export function parseOpenApiSpec(filePath, root, report) {
7
7
  const raw = readFileSync(filePath, "utf8");
8
8
  const lineCounter = new LineCounter();
9
9
  const doc = parseDocument(raw, { lineCounter, keepSourceTokens: true });
10
+ if (report && doc.errors.length > 0)
11
+ report.hadErrors = true;
10
12
  if (!doc.contents)
11
13
  return [];
12
14
  const relPath = relative(root, filePath);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "srclawyer",
3
- "version": "0.6.1",
3
+ "version": "0.6.3",
4
4
  "description": "Static analysis engine that derives privacy policy documentation directly from source code",
5
5
  "license": "SEE LICENSE IN LICENSE",
6
6
  "type": "module",
@@ -38,8 +38,8 @@
38
38
  "@babel/parser": "^7.25.6",
39
39
  "@babel/traverse": "^7.25.6",
40
40
  "commander": "^12.1.0",
41
- "fast-glob": "^3.3.2",
42
41
  "js-yaml": "^4.1.0",
42
+ "tinyglobby": "^0.2.17",
43
43
  "typescript": "^5.6.2",
44
44
  "web-tree-sitter": "^0.27.0",
45
45
  "yaml": "^2.5.1"