srclawyer 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +112 -0
- package/README.md +58 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +32 -0
- package/dist/cloud/config.d.ts +5 -0
- package/dist/cloud/config.js +23 -0
- package/dist/cloud/protectedLogicClient.d.ts +38 -0
- package/dist/cloud/protectedLogicClient.js +71 -0
- package/dist/cloud/resolveProtectedLogic.d.ts +14 -0
- package/dist/cloud/resolveProtectedLogic.js +67 -0
- package/dist/commands/init.d.ts +1 -0
- package/dist/commands/init.js +67 -0
- package/dist/commands/scan.d.ts +6 -0
- package/dist/commands/scan.js +66 -0
- package/dist/config/config.d.ts +18 -0
- package/dist/config/config.js +26 -0
- package/dist/engine/astUtils.d.ts +2 -0
- package/dist/engine/astUtils.js +22 -0
- package/dist/engine/discoverFiles.d.ts +7 -0
- package/dist/engine/discoverFiles.js +25 -0
- package/dist/engine/discoverProjects.d.ts +3 -0
- package/dist/engine/discoverProjects.js +34 -0
- package/dist/engine/languageDetection.d.ts +1 -0
- package/dist/engine/languageDetection.js +48 -0
- package/dist/engine/scanEngine.d.ts +11 -0
- package/dist/engine/scanEngine.js +130 -0
- package/dist/engine/secretRedaction.d.ts +2 -0
- package/dist/engine/secretRedaction.js +31 -0
- package/dist/engine/tsProject.d.ts +4 -0
- package/dist/engine/tsProject.js +68 -0
- package/dist/engine/types.d.ts +71 -0
- package/dist/engine/types.js +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +9 -0
- package/dist/llm/anthropicProvider.d.ts +8 -0
- package/dist/llm/anthropicProvider.js +130 -0
- package/dist/llm/buildRequestItem.d.ts +10 -0
- package/dist/llm/buildRequestItem.js +17 -0
- package/dist/llm/resolveAmbiguousFindings.d.ts +7 -0
- package/dist/llm/resolveAmbiguousFindings.js +103 -0
- package/dist/llm/resolveProviderForTier.d.ts +3 -0
- package/dist/llm/resolveProviderForTier.js +31 -0
- package/dist/llm/types.d.ts +29 -0
- package/dist/llm/types.js +1 -0
- package/dist/policy/dataCategoryLabels.d.ts +2 -0
- package/dist/policy/dataCategoryLabels.js +18 -0
- package/dist/policy/disclaimer.d.ts +1 -0
- package/dist/policy/disclaimer.js +1 -0
- package/dist/policy/generatePolicy.d.ts +3 -0
- package/dist/policy/generatePolicy.js +149 -0
- package/dist/report/formatText.d.ts +2 -0
- package/dist/report/formatText.js +53 -0
- package/dist/rules/confidenceFactors.d.ts +14 -0
- package/dist/rules/confidenceFactors.js +14 -0
- package/dist/rules/dataCategoryLexicon.d.ts +3 -0
- package/dist/rules/dataCategoryLexicon.js +26 -0
- package/dist/rules/frameworkRules.d.ts +5 -0
- package/dist/rules/frameworkRules.js +98 -0
- package/dist/rules/htmlRules.d.ts +6 -0
- package/dist/rules/htmlRules.js +75 -0
- package/dist/rules/processorProfiles.d.ts +2 -0
- package/dist/rules/processorProfiles.js +42 -0
- package/dist/rules/safeFieldNames.d.ts +1 -0
- package/dist/rules/safeFieldNames.js +25 -0
- package/dist/rules/sdkRules.d.ts +3 -0
- package/dist/rules/sdkRules.js +65 -0
- package/dist/rules/typedRequestBodyRules.d.ts +4 -0
- package/dist/rules/typedRequestBodyRules.js +83 -0
- package/dist/rules/webApiRules.d.ts +3 -0
- package/dist/rules/webApiRules.js +89 -0
- package/dist/rules/zodPreFilter.d.ts +9 -0
- package/dist/rules/zodPreFilter.js +91 -0
- package/dist/schemaParsers/graphql.d.ts +2 -0
- package/dist/schemaParsers/graphql.js +52 -0
- package/dist/schemaParsers/openapi.d.ts +2 -0
- package/dist/schemaParsers/openapi.js +60 -0
- package/package.json +51 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import fg from "fast-glob";
|
|
2
|
+
const CODE_PATTERNS = ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx", "**/*.mjs", "**/*.cjs"];
|
|
3
|
+
const HTML_PATTERNS = ["**/*.html", "**/*.htm"];
|
|
4
|
+
const SCHEMA_PATTERNS = ["**/*.graphql", "**/*.gql", "**/openapi.yaml", "**/openapi.yml", "**/openapi.json", "**/swagger.yaml", "**/swagger.yml", "**/swagger.json"];
|
|
5
|
+
const MANIFEST_PATTERNS = ["**/package.json", "**/requirements.txt"];
|
|
6
|
+
const IGNORE = [
|
|
7
|
+
"**/node_modules/**",
|
|
8
|
+
"**/dist/**",
|
|
9
|
+
"**/build/**",
|
|
10
|
+
"**/.next/**",
|
|
11
|
+
"**/coverage/**",
|
|
12
|
+
"**/*.min.js",
|
|
13
|
+
"**/*.test.*",
|
|
14
|
+
"**/*.spec.*",
|
|
15
|
+
"**/*.d.ts",
|
|
16
|
+
];
|
|
17
|
+
export async function discoverFiles(root) {
|
|
18
|
+
const [codeFiles, htmlFiles, schemaFiles, manifestFiles] = await Promise.all([
|
|
19
|
+
fg(CODE_PATTERNS, { cwd: root, absolute: true, ignore: IGNORE, dot: false }),
|
|
20
|
+
fg(HTML_PATTERNS, { cwd: root, absolute: true, ignore: IGNORE, dot: false }),
|
|
21
|
+
fg(SCHEMA_PATTERNS, { cwd: root, absolute: true, ignore: ["**/node_modules/**"], dot: false }),
|
|
22
|
+
fg(MANIFEST_PATTERNS, { cwd: root, absolute: true, ignore: ["**/node_modules/**"], dot: false }),
|
|
23
|
+
]);
|
|
24
|
+
return { codeFiles, htmlFiles, schemaFiles, manifestFiles };
|
|
25
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
2
|
+
import { dirname, join, relative } from "node:path";
|
|
3
|
+
import fg from "fast-glob";
|
|
4
|
+
export async function discoverProjects(root) {
|
|
5
|
+
const rootPackageJsonPath = join(root, "package.json");
|
|
6
|
+
if (!existsSync(rootPackageJsonPath)) {
|
|
7
|
+
return [{ root, kind: "unknown", manifestPath: null }];
|
|
8
|
+
}
|
|
9
|
+
const rootPackageJson = JSON.parse(readFileSync(rootPackageJsonPath, "utf8"));
|
|
10
|
+
const workspacePatterns = Array.isArray(rootPackageJson.workspaces)
|
|
11
|
+
? rootPackageJson.workspaces
|
|
12
|
+
: rootPackageJson.workspaces?.packages;
|
|
13
|
+
const lernaPath = join(root, "lerna.json");
|
|
14
|
+
const nxPath = join(root, "nx.json");
|
|
15
|
+
const hasWorkspaceConfig = Boolean(workspacePatterns) || existsSync(lernaPath) || existsSync(nxPath);
|
|
16
|
+
if (!hasWorkspaceConfig) {
|
|
17
|
+
return [{ root, kind: "node", manifestPath: rootPackageJsonPath }];
|
|
18
|
+
}
|
|
19
|
+
const patterns = workspacePatterns ?? ["packages/*", "apps/*"];
|
|
20
|
+
const manifestPaths = await fg(patterns.map((p) => join(p, "package.json")), { cwd: root, absolute: true, ignore: ["**/node_modules/**"] });
|
|
21
|
+
const projects = manifestPaths.map((manifestPath) => ({
|
|
22
|
+
root: dirname(manifestPath),
|
|
23
|
+
kind: "node",
|
|
24
|
+
manifestPath,
|
|
25
|
+
}));
|
|
26
|
+
if (projects.length === 0) {
|
|
27
|
+
return [{ root, kind: "node", manifestPath: rootPackageJsonPath }];
|
|
28
|
+
}
|
|
29
|
+
return projects;
|
|
30
|
+
}
|
|
31
|
+
export function describeProject(root, project) {
|
|
32
|
+
const rel = relative(root, project.root);
|
|
33
|
+
return rel === "" ? "." : rel;
|
|
34
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function detectUnsupportedStack(root: string): Promise<string | null>;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import fg from "fast-glob";
|
|
4
|
+
const IGNORE = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/.git/**", "**/vendor/**"];
|
|
5
|
+
const SUPPORTED_EXTENSIONS = ["ts", "tsx", "js", "jsx", "mjs", "cjs", "html", "htm"];
|
|
6
|
+
const UNSUPPORTED_LANGUAGES = [
|
|
7
|
+
{ label: "Python", extensions: ["py"] },
|
|
8
|
+
{ label: "Ruby", extensions: ["rb"] },
|
|
9
|
+
{ label: "Go", extensions: ["go"] },
|
|
10
|
+
{ label: "Java", extensions: ["java"] },
|
|
11
|
+
{ label: "PHP", extensions: ["php"] },
|
|
12
|
+
{ label: "C#", extensions: ["cs"] },
|
|
13
|
+
];
|
|
14
|
+
function bannerize(message) {
|
|
15
|
+
return `UNSUPPORTED CODEBASE — ${message} SrcLawyer only analyzes JavaScript/TypeScript and HTML today; the findings below (if any) are incomplete and MUST NOT be treated as a full compliance scan.`;
|
|
16
|
+
}
|
|
17
|
+
async function detectFrameworkSignature(root) {
|
|
18
|
+
if (existsSync(join(root, "manage.py"))) {
|
|
19
|
+
return bannerize("Django detected (manage.py found at the project root).");
|
|
20
|
+
}
|
|
21
|
+
const hasGemfile = existsSync(join(root, "Gemfile"));
|
|
22
|
+
const hasRailsLayout = existsSync(join(root, "config", "routes.rb")) || existsSync(join(root, "app", "controllers"));
|
|
23
|
+
if (hasGemfile && hasRailsLayout) {
|
|
24
|
+
return bannerize("Rails detected (Gemfile + a Rails app/config layout found).");
|
|
25
|
+
}
|
|
26
|
+
const protoFiles = await fg("**/*.proto", { cwd: root, ignore: IGNORE, dot: false });
|
|
27
|
+
if (protoFiles.length > 0) {
|
|
28
|
+
return bannerize(`gRPC/Protocol Buffers detected (${protoFiles.length} .proto file(s) found) — protobuf schemas are not parsed.`);
|
|
29
|
+
}
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
async function detectDominantUnsupportedLanguage(root) {
|
|
33
|
+
const [supportedFiles, ...unsupportedGroups] = await Promise.all([
|
|
34
|
+
fg(SUPPORTED_EXTENSIONS.map((ext) => `**/*.${ext}`), { cwd: root, ignore: IGNORE, dot: false }),
|
|
35
|
+
...UNSUPPORTED_LANGUAGES.map((lang) => fg(lang.extensions.map((ext) => `**/*.${ext}`), { cwd: root, ignore: IGNORE, dot: false })),
|
|
36
|
+
]);
|
|
37
|
+
const counts = UNSUPPORTED_LANGUAGES.map((lang, i) => ({ label: lang.label, count: unsupportedGroups[i].length }));
|
|
38
|
+
const dominant = counts.filter((c) => c.count > 0).sort((a, b) => b.count - a.count)[0];
|
|
39
|
+
if (!dominant || dominant.count <= supportedFiles.length)
|
|
40
|
+
return null;
|
|
41
|
+
return bannerize(`${dominant.label} detected as the primary language (${dominant.count} file(s), vs ${supportedFiles.length} supported JS/TS/HTML file(s)).`);
|
|
42
|
+
}
|
|
43
|
+
export async function detectUnsupportedStack(root) {
|
|
44
|
+
const frameworkWarning = await detectFrameworkSignature(root);
|
|
45
|
+
if (frameworkWarning)
|
|
46
|
+
return frameworkWarning;
|
|
47
|
+
return detectDominantUnsupportedLanguage(root);
|
|
48
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ScanResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Pure, local, offline — no network access anywhere in this function or anything it calls. Zod
|
|
4
|
+
* schemas and confidence scoring can no longer be fully resolved here (see pendingZodCandidates and
|
|
5
|
+
* every Finding.confidenceFactors) — resolving those against the protected-logic service is a
|
|
6
|
+
* separate, explicit step in src/cloud/resolveProtectedLogic.ts, called from commands/scan.ts, the
|
|
7
|
+
* same way Layer 2's resolveAmbiguousFindings.ts is a separate step layered on top of this function
|
|
8
|
+
* rather than folded into it. Anyone auditing "does this touch the network" can still answer that by
|
|
9
|
+
* reading this file alone.
|
|
10
|
+
*/
|
|
11
|
+
export declare function scan(root: string): Promise<ScanResult>;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { extname } from "node:path";
|
|
3
|
+
import { discoverProjects } from "./discoverProjects.js";
|
|
4
|
+
import { discoverFiles } from "./discoverFiles.js";
|
|
5
|
+
import { detectUnsupportedStack } from "./languageDetection.js";
|
|
6
|
+
import { parseSource } from "./astUtils.js";
|
|
7
|
+
import { runSdkRules } from "../rules/sdkRules.js";
|
|
8
|
+
import { runFrameworkRules } from "../rules/frameworkRules.js";
|
|
9
|
+
import { runWebApiRules } from "../rules/webApiRules.js";
|
|
10
|
+
import { runTypedRequestBodyRules } from "../rules/typedRequestBodyRules.js";
|
|
11
|
+
import { findZodCandidates } from "../rules/zodPreFilter.js";
|
|
12
|
+
import { buildTsProject } from "./tsProject.js";
|
|
13
|
+
import { runHtmlInputRules, extractInlineScripts } from "../rules/htmlRules.js";
|
|
14
|
+
import { parseOpenApiSpec } from "../schemaParsers/openapi.js";
|
|
15
|
+
import { parseGraphQLSchema } from "../schemaParsers/graphql.js";
|
|
16
|
+
const OPENAPI_NAME = /(openapi|swagger)\.(ya?ml|json)$/i;
|
|
17
|
+
const GRAPHQL_NAME = /\.(graphql|gql)$/i;
|
|
18
|
+
// Built at most once per project root, per scan, and only the first time a rule actually needs it
|
|
19
|
+
// (see typedRequestBodyRules.ts) — measured against the real dub/formbricks golden-corpus repos at
|
|
20
|
+
// ~2.5s/~800MB and ~2.2s/~650MB respectively for a full ts.Program build. Eager construction would tax
|
|
21
|
+
// every scan of a large project regardless of whether it ever hits the one pattern that needs it;
|
|
22
|
+
// memoized-lazy pays that cost at most once, only on the scan that actually earns it.
|
|
23
|
+
function makeLazyTsProject(files) {
|
|
24
|
+
let handle;
|
|
25
|
+
return () => {
|
|
26
|
+
if (handle === undefined) {
|
|
27
|
+
try {
|
|
28
|
+
handle = buildTsProject(files);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
handle = null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return handle;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function runCodeRules(filePath, source, root, getTsProject, zodCandidates) {
|
|
38
|
+
const ast = parseSource(source, filePath);
|
|
39
|
+
zodCandidates.push(...findZodCandidates(source, filePath, root));
|
|
40
|
+
if (!ast)
|
|
41
|
+
return [];
|
|
42
|
+
return [
|
|
43
|
+
...runSdkRules(ast, filePath, source, root),
|
|
44
|
+
...runFrameworkRules(ast, filePath, source, root),
|
|
45
|
+
...runWebApiRules(ast, filePath, source, root),
|
|
46
|
+
...runTypedRequestBodyRules(ast, filePath, source, root, getTsProject),
|
|
47
|
+
];
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Pure, local, offline — no network access anywhere in this function or anything it calls. Zod
|
|
51
|
+
* schemas and confidence scoring can no longer be fully resolved here (see pendingZodCandidates and
|
|
52
|
+
* every Finding.confidenceFactors) — resolving those against the protected-logic service is a
|
|
53
|
+
* separate, explicit step in src/cloud/resolveProtectedLogic.ts, called from commands/scan.ts, the
|
|
54
|
+
* same way Layer 2's resolveAmbiguousFindings.ts is a separate step layered on top of this function
|
|
55
|
+
* rather than folded into it. Anyone auditing "does this touch the network" can still answer that by
|
|
56
|
+
* reading this file alone.
|
|
57
|
+
*/
|
|
58
|
+
export async function scan(root) {
|
|
59
|
+
const [projects, unsupportedStackWarning] = await Promise.all([discoverProjects(root), detectUnsupportedStack(root)]);
|
|
60
|
+
const findings = [];
|
|
61
|
+
const pendingZodCandidates = [];
|
|
62
|
+
let filesScanned = 0;
|
|
63
|
+
const projectRootList = [...new Set(projects.map((p) => p.root))];
|
|
64
|
+
const fileSets = await Promise.all(projectRootList.map((projectRoot) => discoverFiles(projectRoot)));
|
|
65
|
+
const allCodeFiles = new Set();
|
|
66
|
+
const allHtmlFiles = new Set();
|
|
67
|
+
const allSchemaFiles = new Set();
|
|
68
|
+
const rootForFile = new Map();
|
|
69
|
+
const tsProjectByRoot = new Map();
|
|
70
|
+
fileSets.forEach((set, i) => {
|
|
71
|
+
const projectRoot = projectRootList[i];
|
|
72
|
+
tsProjectByRoot.set(projectRoot, makeLazyTsProject(set.codeFiles));
|
|
73
|
+
set.codeFiles.forEach((f) => {
|
|
74
|
+
allCodeFiles.add(f);
|
|
75
|
+
rootForFile.set(f, projectRoot);
|
|
76
|
+
});
|
|
77
|
+
set.htmlFiles.forEach((f) => allHtmlFiles.add(f));
|
|
78
|
+
set.schemaFiles.forEach((f) => allSchemaFiles.add(f));
|
|
79
|
+
});
|
|
80
|
+
// HTML files (and their inline <script> blocks) aren't part of any codeFiles list and never carry
|
|
81
|
+
// a resolvable TS type, so they get a permanently-null thunk rather than a project association.
|
|
82
|
+
const noTsProject = () => null;
|
|
83
|
+
for (const filePath of allCodeFiles) {
|
|
84
|
+
filesScanned += 1;
|
|
85
|
+
let source;
|
|
86
|
+
try {
|
|
87
|
+
source = readFileSync(filePath, "utf8");
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
const getTsProject = tsProjectByRoot.get(rootForFile.get(filePath) ?? "") ?? noTsProject;
|
|
93
|
+
findings.push(...runCodeRules(filePath, source, root, getTsProject, pendingZodCandidates));
|
|
94
|
+
}
|
|
95
|
+
for (const filePath of allHtmlFiles) {
|
|
96
|
+
filesScanned += 1;
|
|
97
|
+
let html;
|
|
98
|
+
try {
|
|
99
|
+
html = readFileSync(filePath, "utf8");
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
findings.push(...runHtmlInputRules(html, filePath, root));
|
|
105
|
+
for (const script of extractInlineScripts(html)) {
|
|
106
|
+
findings.push(...runCodeRules(filePath, script.code, root, noTsProject, pendingZodCandidates));
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
for (const filePath of allSchemaFiles) {
|
|
110
|
+
filesScanned += 1;
|
|
111
|
+
if (OPENAPI_NAME.test(filePath)) {
|
|
112
|
+
try {
|
|
113
|
+
findings.push(...parseOpenApiSpec(filePath, root));
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
else if (GRAPHQL_NAME.test(filePath) || extname(filePath) === ".graphql") {
|
|
120
|
+
try {
|
|
121
|
+
findings.push(...parseGraphQLSchema(filePath, root));
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const ambiguousCount = findings.filter((f) => f.requiresReview).length;
|
|
129
|
+
return { projects, findings, ambiguousCount, filesScanned, unsupportedStackWarning, pendingZodCandidates };
|
|
130
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
const SECRET_PATTERNS = [
|
|
2
|
+
/sk_live_[A-Za-z0-9]{16,}/g,
|
|
3
|
+
/sk_test_[A-Za-z0-9]{16,}/g,
|
|
4
|
+
/pk_live_[A-Za-z0-9]{16,}/g,
|
|
5
|
+
// Anthropic API keys — matched by shape alone (like the patterns above), not by the surrounding
|
|
6
|
+
// variable name or quoting, so `key = "sk-ant-..."` and an unquoted `KEY=sk-ant-...` both redact.
|
|
7
|
+
/sk-ant-[A-Za-z0-9_-]{20,}/g,
|
|
8
|
+
/AKIA[0-9A-Z]{16}/g,
|
|
9
|
+
/AIza[0-9A-Za-z\-_]{35}/g,
|
|
10
|
+
/ghp_[A-Za-z0-9]{36}/g,
|
|
11
|
+
/xox[baprs]-[A-Za-z0-9-]{10,}/g,
|
|
12
|
+
/SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}/g,
|
|
13
|
+
/eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/g,
|
|
14
|
+
/postgres(?:ql)?:\/\/[^\s"'`]+/gi,
|
|
15
|
+
/mongodb(?:\+srv)?:\/\/[^\s"'`]+/gi,
|
|
16
|
+
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----/g,
|
|
17
|
+
/(?:api[_-]?key|secret|token|password|passwd)\s*[:=]\s*["'`][^"'`\s]{12,}["'`]/gi,
|
|
18
|
+
];
|
|
19
|
+
export function redactSecrets(source) {
|
|
20
|
+
let redacted = source;
|
|
21
|
+
for (const pattern of SECRET_PATTERNS) {
|
|
22
|
+
redacted = redacted.replace(pattern, "[REDACTED]");
|
|
23
|
+
}
|
|
24
|
+
return redacted;
|
|
25
|
+
}
|
|
26
|
+
export function containsLikelySecret(source) {
|
|
27
|
+
return SECRET_PATTERNS.some((pattern) => {
|
|
28
|
+
pattern.lastIndex = 0;
|
|
29
|
+
return pattern.test(source);
|
|
30
|
+
});
|
|
31
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import ts from "typescript";
|
|
2
|
+
// Purely defensive, not a tuned threshold: the largest repo actually measured (dub, 4211 files) took
|
|
3
|
+
// ~2.5s/~800MB to build. This is ~5x that, so a project this large skips type resolution entirely
|
|
4
|
+
// rather than attempting an untested, unbounded-cost ts.createProgram call.
|
|
5
|
+
const MAX_PROGRAM_FILES = 20_000;
|
|
6
|
+
const NULL_HANDLE = {
|
|
7
|
+
getObjectTypeMembersAt: () => null,
|
|
8
|
+
};
|
|
9
|
+
function findNodeAtPosition(root, position) {
|
|
10
|
+
if (position < root.getFullStart() || position >= root.getEnd()) {
|
|
11
|
+
return root.getStart() <= position && position < root.getEnd() ? root : undefined;
|
|
12
|
+
}
|
|
13
|
+
let found;
|
|
14
|
+
root.forEachChild((child) => {
|
|
15
|
+
if (found)
|
|
16
|
+
return;
|
|
17
|
+
if (position >= child.getFullStart() && position < child.getEnd()) {
|
|
18
|
+
found = findNodeAtPosition(child, position) ?? child;
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
return found ?? (root.getStart() <= position && position < root.getEnd() ? root : undefined);
|
|
22
|
+
}
|
|
23
|
+
export function buildTsProject(files) {
|
|
24
|
+
if (files.length === 0 || files.length > MAX_PROGRAM_FILES)
|
|
25
|
+
return NULL_HANDLE;
|
|
26
|
+
let program;
|
|
27
|
+
let checker;
|
|
28
|
+
try {
|
|
29
|
+
program = ts.createProgram(files, {
|
|
30
|
+
allowJs: true,
|
|
31
|
+
checkJs: false,
|
|
32
|
+
noEmit: true,
|
|
33
|
+
skipLibCheck: true,
|
|
34
|
+
});
|
|
35
|
+
checker = program.getTypeChecker();
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return NULL_HANDLE;
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
getObjectTypeMembersAt(filePath, line, column) {
|
|
42
|
+
try {
|
|
43
|
+
const sourceFile = program.getSourceFile(filePath);
|
|
44
|
+
if (!sourceFile)
|
|
45
|
+
return null;
|
|
46
|
+
const position = ts.getPositionOfLineAndCharacter(sourceFile, line - 1, column);
|
|
47
|
+
const node = findNodeAtPosition(sourceFile, position);
|
|
48
|
+
if (!node)
|
|
49
|
+
return null;
|
|
50
|
+
const type = checker.getTypeAtLocation(node);
|
|
51
|
+
// Primitives (number/string/boolean) have non-empty "properties" too — their boxed prototype
|
|
52
|
+
// methods (toFixed, toUpperCase, ...) — which getPropertiesOfType happily returns. Only a real
|
|
53
|
+
// object type (an interface, a type literal, a class) is a genuine request-body shape.
|
|
54
|
+
if (!(type.flags & ts.TypeFlags.Object))
|
|
55
|
+
return null;
|
|
56
|
+
const properties = checker.getPropertiesOfType(type);
|
|
57
|
+
if (properties.length === 0)
|
|
58
|
+
return null;
|
|
59
|
+
return properties.map((p) => p.getName());
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
// Never let a target project's own type errors, an out-of-range position, or any other
|
|
63
|
+
// resolution failure break the scan — an honest "couldn't resolve this" is a null, not a throw.
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
export type DataCategory = "email" | "name" | "phone" | "physical_address" | "payment_info" | "government_id" | "device_id" | "ip_address" | "location" | "analytics_usage" | "authentication_credentials" | "childrens_data" | "health_data" | "microphone_audio" | "camera_video" | "generic_pii";
|
|
2
|
+
export type Confidence = "high" | "medium" | "low";
|
|
3
|
+
export type ResolutionOutcome = "resolved" | "declined" | "unresolved";
|
|
4
|
+
export interface LlmResolution {
|
|
5
|
+
outcome: ResolutionOutcome;
|
|
6
|
+
reason: string | null;
|
|
7
|
+
provider: string;
|
|
8
|
+
model: string | null;
|
|
9
|
+
resolvedDataCategories?: DataCategory[];
|
|
10
|
+
resolvedDescription?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface CodeLocation {
|
|
13
|
+
file: string;
|
|
14
|
+
line: number;
|
|
15
|
+
column: number;
|
|
16
|
+
}
|
|
17
|
+
export interface Finding {
|
|
18
|
+
id: string;
|
|
19
|
+
dataCategories: DataCategory[];
|
|
20
|
+
processor: string | null;
|
|
21
|
+
description: string;
|
|
22
|
+
confidence: Confidence;
|
|
23
|
+
source: "sdk-rule" | "framework-rule" | "web-api-rule" | "html-input-rule" | "openapi" | "graphql" | "zod-rule" | "type-rule";
|
|
24
|
+
location: CodeLocation;
|
|
25
|
+
evidence: string;
|
|
26
|
+
requiresReview: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* Present only for findings that were run through Layer 2 resolution.
|
|
29
|
+
* Absent means: free tier, --no-llm, or requiresReview was already false —
|
|
30
|
+
* this finding was never sent to an LLM at all.
|
|
31
|
+
*/
|
|
32
|
+
llmResolution?: LlmResolution;
|
|
33
|
+
/**
|
|
34
|
+
* Present ONLY for findings whose confidence/requiresReview are provisional, pending the
|
|
35
|
+
* protected-logic scoring call (src/cloud/protectedLogicClient.ts) — the fields captured here are
|
|
36
|
+
* exactly what that endpoint needs (field names and derived yes/no signals, never source code).
|
|
37
|
+
* Until scored, `confidence`/`requiresReview` hold the safe fallback ("low"/true). Absent entirely
|
|
38
|
+
* for findings whose confidence is fixed locally (sdkRules, webApiRules, htmlRules, schema parsers)
|
|
39
|
+
* and never sent anywhere.
|
|
40
|
+
*/
|
|
41
|
+
confidenceFactors?: Record<string, boolean | number | string | null>;
|
|
42
|
+
}
|
|
43
|
+
export interface ProcessorProfile {
|
|
44
|
+
name: string;
|
|
45
|
+
dataCategories: DataCategory[];
|
|
46
|
+
disclosureNotes: string;
|
|
47
|
+
}
|
|
48
|
+
export interface ProjectManifest {
|
|
49
|
+
root: string;
|
|
50
|
+
kind: "node" | "unknown";
|
|
51
|
+
manifestPath: string | null;
|
|
52
|
+
}
|
|
53
|
+
export interface ScanResult {
|
|
54
|
+
projects: ProjectManifest[];
|
|
55
|
+
findings: Finding[];
|
|
56
|
+
ambiguousCount: number;
|
|
57
|
+
filesScanned: number;
|
|
58
|
+
unsupportedStackWarning: string | null;
|
|
59
|
+
/**
|
|
60
|
+
* Files that look like they might define a Zod schema used to validate incoming request data —
|
|
61
|
+
* unconfirmed, since the actual schema/anchor detection is protected logic and runs server-side.
|
|
62
|
+
* Resolved (or, on failure, honestly left unresolved) by resolveProtectedLogic() in
|
|
63
|
+
* commands/scan.ts, exactly like requiresReview findings are resolved by Layer 2's
|
|
64
|
+
* resolveAmbiguousFindings() — not inside scan() itself, which stays pure/offline.
|
|
65
|
+
*/
|
|
66
|
+
pendingZodCandidates: Array<{
|
|
67
|
+
id: string;
|
|
68
|
+
codeFragment: string;
|
|
69
|
+
fragmentLineToSourceLine: number[];
|
|
70
|
+
}>;
|
|
71
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public library surface for `srclawyer`.
|
|
3
|
+
*
|
|
4
|
+
* This is the ONLY supported way for other packages (including the private detection-rules
|
|
5
|
+
* package) to depend on this repo's code. Everything else under src/ is an internal implementation
|
|
6
|
+
* detail and may change without notice — import from here, never from a deep relative/dist path.
|
|
7
|
+
*/
|
|
8
|
+
export { redactSecrets, containsLikelySecret } from "./engine/secretRedaction.js";
|
|
9
|
+
export { parseSource } from "./engine/astUtils.js";
|
|
10
|
+
export type { DataCategory, Confidence, ResolutionOutcome, LlmResolution, CodeLocation, Finding, ProcessorProfile, ProjectManifest, ScanResult, } from "./engine/types.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public library surface for `srclawyer`.
|
|
3
|
+
*
|
|
4
|
+
* This is the ONLY supported way for other packages (including the private detection-rules
|
|
5
|
+
* package) to depend on this repo's code. Everything else under src/ is an internal implementation
|
|
6
|
+
* detail and may change without notice — import from here, never from a deep relative/dist path.
|
|
7
|
+
*/
|
|
8
|
+
export { redactSecrets, containsLikelySecret } from "./engine/secretRedaction.js";
|
|
9
|
+
export { parseSource } from "./engine/astUtils.js";
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { LlmBatchRequest, LlmProvider, LlmResponseItem } from "./types.js";
|
|
2
|
+
export declare class AnthropicProvider implements LlmProvider {
|
|
3
|
+
readonly name = "anthropic";
|
|
4
|
+
readonly model: string;
|
|
5
|
+
private readonly apiKey;
|
|
6
|
+
constructor(apiKey: string, model?: string);
|
|
7
|
+
resolveBatch(request: LlmBatchRequest): Promise<LlmResponseItem[]>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { DATA_CATEGORY_LABELS } from "../policy/dataCategoryLabels.js";
|
|
2
|
+
const ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages";
|
|
3
|
+
const ANTHROPIC_VERSION = "2023-06-01";
|
|
4
|
+
// Confirm this is still current at implementation/release time.
|
|
5
|
+
const DEFAULT_MODEL = "claude-sonnet-5";
|
|
6
|
+
const REQUEST_TIMEOUT_MS = 30_000;
|
|
7
|
+
// Same source of truth resolveAmbiguousFindings.ts validates responses against (KNOWN_CATEGORIES) —
|
|
8
|
+
// constraining the tool schema to this enum means a legitimate resolution can no longer be discarded
|
|
9
|
+
// as "unrecognized category" just because the model wasn't told what vocabulary to stay inside.
|
|
10
|
+
const DATA_CATEGORY_VALUES = Object.keys(DATA_CATEGORY_LABELS);
|
|
11
|
+
const RESOLUTION_TOOL = {
|
|
12
|
+
name: "report_resolutions",
|
|
13
|
+
description: "Report a classification decision for each numbered data-collection finding provided.",
|
|
14
|
+
input_schema: {
|
|
15
|
+
type: "object",
|
|
16
|
+
properties: {
|
|
17
|
+
resolutions: {
|
|
18
|
+
type: "array",
|
|
19
|
+
items: {
|
|
20
|
+
type: "object",
|
|
21
|
+
properties: {
|
|
22
|
+
findingId: { type: "string" },
|
|
23
|
+
outcome: { type: "string", enum: ["resolved", "declined"] },
|
|
24
|
+
reason: {
|
|
25
|
+
type: ["string", "null"],
|
|
26
|
+
description: 'Required when outcome is "declined": explain concretely why this could not be confidently classified. Null when resolved.',
|
|
27
|
+
},
|
|
28
|
+
resolvedDataCategories: {
|
|
29
|
+
type: "array",
|
|
30
|
+
items: { type: "string", enum: DATA_CATEGORY_VALUES },
|
|
31
|
+
description: "Only when outcome is \"resolved\". Must use these exact category values — never a category outside this list.",
|
|
32
|
+
},
|
|
33
|
+
resolvedDescription: { type: "string" },
|
|
34
|
+
},
|
|
35
|
+
required: ["findingId", "outcome", "reason"],
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
required: ["resolutions"],
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
function buildSystemPrompt() {
|
|
43
|
+
return [
|
|
44
|
+
"You are assisting a static-analysis privacy-compliance tool. You will be given a list of code-derived findings that automated pattern matching could not confidently classify into a personal-data category.",
|
|
45
|
+
"For each item, decide whether you can confidently classify what kind of personal data it represents based ONLY on the redacted evidence and description given — do not assume context that isn't shown.",
|
|
46
|
+
'If confident, set outcome to "resolved" and provide resolvedDataCategories and a short resolvedDescription.',
|
|
47
|
+
'If not confident, set outcome to "declined" and give a concrete, specific reason — never guess.',
|
|
48
|
+
'Any item marked isHighStakes must ALWAYS be "declined" with a reason noting it requires manual/legal review, regardless of how confident you feel — this is a hard rule, not a suggestion.',
|
|
49
|
+
"Respond only by calling the report_resolutions tool with exactly one entry per item given, in the same findingId order.",
|
|
50
|
+
].join(" ");
|
|
51
|
+
}
|
|
52
|
+
export class AnthropicProvider {
|
|
53
|
+
name = "anthropic";
|
|
54
|
+
model;
|
|
55
|
+
apiKey;
|
|
56
|
+
constructor(apiKey, model = DEFAULT_MODEL) {
|
|
57
|
+
this.apiKey = apiKey;
|
|
58
|
+
this.model = model;
|
|
59
|
+
}
|
|
60
|
+
async resolveBatch(request) {
|
|
61
|
+
const userContent = JSON.stringify(request.items.map((item) => ({
|
|
62
|
+
findingId: item.findingId,
|
|
63
|
+
dataCategories: item.dataCategories,
|
|
64
|
+
isHighStakes: item.isHighStakes,
|
|
65
|
+
evidence: item.redactedEvidence,
|
|
66
|
+
description: item.redactedDescription,
|
|
67
|
+
})), null, 2);
|
|
68
|
+
const controller = new AbortController();
|
|
69
|
+
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
|
70
|
+
let response;
|
|
71
|
+
try {
|
|
72
|
+
response = await fetch(ANTHROPIC_API_URL, {
|
|
73
|
+
method: "POST",
|
|
74
|
+
headers: {
|
|
75
|
+
"x-api-key": this.apiKey,
|
|
76
|
+
"anthropic-version": ANTHROPIC_VERSION,
|
|
77
|
+
"content-type": "application/json",
|
|
78
|
+
},
|
|
79
|
+
body: JSON.stringify({
|
|
80
|
+
model: this.model,
|
|
81
|
+
max_tokens: 4096,
|
|
82
|
+
system: buildSystemPrompt(),
|
|
83
|
+
messages: [{ role: "user", content: userContent }],
|
|
84
|
+
tools: [RESOLUTION_TOOL],
|
|
85
|
+
tool_choice: { type: "tool", name: "report_resolutions" },
|
|
86
|
+
}),
|
|
87
|
+
signal: controller.signal,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
clearTimeout(timeout);
|
|
92
|
+
}
|
|
93
|
+
if (!response.ok) {
|
|
94
|
+
const body = await response.text().catch(() => "");
|
|
95
|
+
throw new Error(`Anthropic API request failed (${response.status}): ${body.slice(0, 200)}`);
|
|
96
|
+
}
|
|
97
|
+
const payload = (await response.json());
|
|
98
|
+
const toolUse = payload.content?.find((block) => block.type === "tool_use" && block.name === "report_resolutions");
|
|
99
|
+
if (!toolUse || typeof toolUse.input !== "object" || toolUse.input === null) {
|
|
100
|
+
throw new Error("Anthropic response did not include a valid report_resolutions tool call");
|
|
101
|
+
}
|
|
102
|
+
const input = toolUse.input;
|
|
103
|
+
if (!Array.isArray(input.resolutions)) {
|
|
104
|
+
throw new Error("Anthropic tool response missing a resolutions array");
|
|
105
|
+
}
|
|
106
|
+
return input.resolutions.map(normalizeResponseItem).filter((item) => item !== null);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// Anthropic's tool-use output reliably omits `reason` entirely for a "resolved" item — despite it being
|
|
110
|
+
// listed in `required` — rather than sending an explicit `null` (confirmed against a real response during
|
|
111
|
+
// live verification on 2026-09-13). Treating an omitted key as invalid, rather than as the `null` the field
|
|
112
|
+
// means when resolved, was silently discarding otherwise-correct resolutions as "malformed."
|
|
113
|
+
function normalizeResponseItem(value) {
|
|
114
|
+
if (typeof value !== "object" || value === null)
|
|
115
|
+
return null;
|
|
116
|
+
const item = value;
|
|
117
|
+
if (typeof item.findingId !== "string")
|
|
118
|
+
return null;
|
|
119
|
+
if (item.outcome !== "resolved" && item.outcome !== "declined")
|
|
120
|
+
return null;
|
|
121
|
+
if (item.reason !== null && item.reason !== undefined && typeof item.reason !== "string")
|
|
122
|
+
return null;
|
|
123
|
+
return {
|
|
124
|
+
findingId: item.findingId,
|
|
125
|
+
outcome: item.outcome,
|
|
126
|
+
reason: item.reason ?? null,
|
|
127
|
+
resolvedDataCategories: item.resolvedDataCategories,
|
|
128
|
+
resolvedDescription: item.resolvedDescription,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Finding } from "../engine/types.js";
|
|
2
|
+
import type { LlmRequestItem } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* The only function permitted to read finding.evidence/description for
|
|
5
|
+
* Layer 2 purposes. Both fields are passed through redactSecrets() before
|
|
6
|
+
* they leave this function — LlmRequestItem has no raw-evidence field, so a
|
|
7
|
+
* provider implementation is structurally incapable of receiving
|
|
8
|
+
* unredacted content.
|
|
9
|
+
*/
|
|
10
|
+
export declare function buildRequestItem(finding: Finding, collectsChildrensData: boolean): LlmRequestItem;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { redactSecrets } from "../engine/secretRedaction.js";
|
|
2
|
+
/**
|
|
3
|
+
* The only function permitted to read finding.evidence/description for
|
|
4
|
+
* Layer 2 purposes. Both fields are passed through redactSecrets() before
|
|
5
|
+
* they leave this function — LlmRequestItem has no raw-evidence field, so a
|
|
6
|
+
* provider implementation is structurally incapable of receiving
|
|
7
|
+
* unredacted content.
|
|
8
|
+
*/
|
|
9
|
+
export function buildRequestItem(finding, collectsChildrensData) {
|
|
10
|
+
return {
|
|
11
|
+
findingId: finding.id,
|
|
12
|
+
redactedEvidence: redactSecrets(finding.evidence),
|
|
13
|
+
redactedDescription: redactSecrets(finding.description),
|
|
14
|
+
dataCategories: finding.dataCategories,
|
|
15
|
+
isHighStakes: finding.dataCategories.includes("childrens_data") || collectsChildrensData,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Finding } from "../engine/types.js";
|
|
2
|
+
import type { LlmProvider } from "./types.js";
|
|
3
|
+
export interface ResolveAmbiguousFindingsOptions {
|
|
4
|
+
collectsChildrensData: boolean;
|
|
5
|
+
batchSize?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function resolveAmbiguousFindings(findings: Finding[], provider: LlmProvider, options: ResolveAmbiguousFindingsOptions): Promise<Finding[]>;
|