srclawyer 0.4.1 → 0.6.0

package/dist/rules/pythonSdkRules.js

@@ -0,0 +1,73 @@
+import { relative } from "node:path";
+import { PROCESSOR_PROFILES } from "./processorProfiles.js";
+import { redactSecrets } from "../engine/secretRedaction.js";
+/**
+ * Deliberately narrower than sdkRules.ts's JS table -- not every processor has a reliable
+ * import-name signal in Python's ecosystem:
+ *   - Cognito: Python code almost always calls it through boto3, AWS's *general* SDK used for every
+ *     AWS service (S3, DynamoDB, SQS, ...). Mapping a bare `import boto3` to "Amazon Cognito" would be
+ *     wrong far more often than right; reliably detecting it needs finding the specific
+ *     `boto3.client("cognito-idp")` call, not just the import -- a real gap, not covered here.
+ *   - Google Analytics: no single well-known, unambiguous pip import root the way `stripe` or
+ *     `twilio` are for their processors. Left out rather than guessed.
+ */
+const SDK_MATCHERS = [
+    { processorKey: "stripe", moduleNames: ["stripe"] },
+    { processorKey: "sendgrid", moduleNames: ["sendgrid"] },
+    { processorKey: "twilio", moduleNames: ["twilio"] },
+    { processorKey: "mixpanel", moduleNames: ["mixpanel"] },
+    { processorKey: "auth0", moduleNames: ["auth0"] },
+    // PyPI package is "firebase-admin"; its import root is firebase_admin.
+    { processorKey: "firebase", moduleNames: ["firebase_admin"] },
+];
+function matchProcessor(moduleRoot) {
+    return SDK_MATCHERS.find((matcher) => matcher.moduleNames.includes(moduleRoot));
+}
+/** The root package name from a dotted module path node (e.g. "twilio" from "twilio.rest"). */
+function moduleRootName(moduleNameNode) {
+    if (!moduleNameNode)
+        return null;
+    if (moduleNameNode.type === "dotted_name") {
+        return moduleNameNode.child(0)?.text ?? null;
+    }
+    if (moduleNameNode.type === "aliased_import") {
+        return moduleRootName(moduleNameNode.childForFieldName("name"));
+    }
+    // relative_import ("from . import x") and anything else has no external package name.
+    return null;
+}
+export function runPythonSdkRules(tree, filePath, source, root) {
+    const findings = [];
+    const lines = source.split("\n");
+    const relPath = relative(root, filePath);
+    const seenProcessors = new Set();
+    const importNodes = [
+        ...tree.rootNode.descendantsOfType("import_statement"),
+        ...tree.rootNode.descendantsOfType("import_from_statement"),
+    ];
+    for (const node of importNodes) {
+        const fieldName = node.type === "import_statement" ? "name" : "module_name";
+        const moduleRoot = moduleRootName(node.childForFieldName(fieldName));
+        if (!moduleRoot)
+            continue;
+        const matcher = matchProcessor(moduleRoot);
+        if (!matcher || seenProcessors.has(matcher.processorKey))
+            continue;
+        seenProcessors.add(matcher.processorKey);
+        const profile = PROCESSOR_PROFILES[matcher.processorKey];
+        const line = node.startPosition.row + 1;
+        const evidenceLine = lines[line - 1] ?? "";
+        findings.push({
+            id: `sdk:${matcher.processorKey}:${relPath}:${line}`,
+            dataCategories: profile.dataCategories,
+            processor: profile.name,
+            description: `${profile.name} integration detected. ${profile.disclosureNotes}`,
+            confidence: "high",
+            source: "sdk-rule",
+            location: { file: relPath, line, column: 0 },
+            evidence: redactSecrets(evidenceLine.trim()),
+            requiresReview: false,
+        });
+    }
+    return findings;
+}

package/dist/rules/safeFieldNames.js

@@ -1,3 +1,9 @@
+// Exact-match only, deliberately narrow -- this is the one silent-drop path in the whole pipeline
+// (a name on this list never becomes a finding at all, not even a low-confidence one flagged for
+// review). Names that are ALSO plausible PII in some real API (e.g. "next"/"prev" as redirect
+// targets, "since"/"until"/"before"/"after" as ambiguous short words) are deliberately left off even
+// when they're common pagination/cursor names elsewhere, so they stay visible for manual review
+// rather than being silently and permanently suppressed.
 const SAFE_FIELD_NAMES = new Set([
     "id",
     "uuid",
@@ -16,6 +22,14 @@ const SAFE_FIELD_NAMES = new Set([
     "total",
     "sort",
     "order",
+    "perpage",
+    "pagesize",
+    "cursor",
+    "nextcursor",
+    "prevcursor",
+    "sortby",
+    "orderby",
+    "sortorder",
 ]);
 function normalize(fieldName) {
     return fieldName.toLowerCase().replace(/[^a-z0-9]/g, "");

package/dist/rules/zodPreFilter.d.ts

@@ -1,3 +1,17 @@
+/**
+ * A hard ceiling, not a normal operating limit: protectedLogicClient.ts batches by total bytes, so a
+ * large fragment is sent ALONE in its own request rather than bundled with others -- the actual
+ * per-batch CPU risk that caused intermittent "Worker exceeded CPU time limit" failures (confirmed via
+ * wrangler tail) is handled there, not here. This constant exists only to refuse something pathological
+ * (e.g. a machine-generated file with one enormous schema), not to cap ordinary real-world files -- and
+ * trimmed extraction (above) means ordinary files need it even less than before.
+ *
+ * Measured (uncapped, pre-trimming) across the three golden-corpus repos: max 41,992 bytes
+ * (formbricks/packages/types/surveys/types.ts). This value is PROVISIONAL and unvalidated against the
+ * real Worker limit: that depends on a Cloudflare plan upgrade that hasn't happened yet. Revisit once
+ * the upgrade is confirmed and the 10-run wrangler tail test has been repeated.
+ */
+export declare const MAX_FRAGMENT_BYTES = 65536;
 export interface ZodCandidate {
     id: string;
     codeFragment: string;
@@ -6,4 +20,10 @@ export interface ZodCandidate {
      *  since the fragment is a non-contiguous excerpt of the original file, not a raw slice. */
     fragmentLineToSourceLine: number[];
 }
-export declare function findZodCandidates(source: string, filePath: string, root: string): ZodCandidate[];
+export interface FindZodCandidatesResult {
+    candidates: ZodCandidate[];
+    /** Relative paths of files whose extracted fragment exceeded MAX_FRAGMENT_BYTES and was dropped
+     *  rather than sent -- the caller must surface these, never drop them silently. */
+    oversizedFiles: string[];
+}
+export declare function findZodCandidates(source: string, filePath: string, root: string): FindZodCandidatesResult;

package/dist/rules/zodPreFilter.js

@@ -1,6 +1,8 @@
 import { relative } from "node:path";
+import _traverse from "@babel/traverse";
 import { redactSecrets } from "../engine/secretRedaction.js";
 import { parseSource } from "../engine/astUtils.js";
+const traverse = (_traverse.default ?? _traverse);
 /**
  * The real Zod schema/anchor detection (which call sites count as "this validates incoming request
  * data" vs. a third-party API response or an env-var schema — the actual scope-boundary reasoning)
@@ -16,9 +18,30 @@ import { parseSource } from "../engine/astUtils.js";
  * statements instead guarantees every included chunk is independently valid, so concatenating a subset
  * of them is always a valid Program — no re-implementation of the protected scope-boundary reasoning
  * is needed to get that guarantee, just "which top-level statement is this line part of."
+ *
+ * Within the anchor's own enclosing statement, any NESTED function body that doesn't itself contain an
+ * anchor match is elided down to an empty block (`{}`) rather than sent whole: the schema judgment only
+ * needs the schema and the shape of the anchor call (`X.inputSchema(Y)` / `Y.parse(...)`), never the
+ * unrelated business logic a real handler wraps around it (formbricks' signup action alone was ~130
+ * lines of that). Eliding is itself statement-boundary-based (a whole function body, start to end), so
+ * it can't produce invalid syntax the way an arbitrary line cut could.
  */
 const ZOD_IMPORT_RE = /from\s+["']zod(?:\/v[34](?:-mini)?)?["']/;
 const ANCHOR_CALL_RE = /\.(?:parse|safeParse|parseAsync|safeParseAsync|inputSchema)\s*\(/;
+/**
+ * A hard ceiling, not a normal operating limit: protectedLogicClient.ts batches by total bytes, so a
+ * large fragment is sent ALONE in its own request rather than bundled with others -- the actual
+ * per-batch CPU risk that caused intermittent "Worker exceeded CPU time limit" failures (confirmed via
+ * wrangler tail) is handled there, not here. This constant exists only to refuse something pathological
+ * (e.g. a machine-generated file with one enormous schema), not to cap ordinary real-world files -- and
+ * trimmed extraction (above) means ordinary files need it even less than before.
+ *
+ * Measured (uncapped, pre-trimming) across the three golden-corpus repos: max 41,992 bytes
+ * (formbricks/packages/types/surveys/types.ts). This value is PROVISIONAL and unvalidated against the
+ * real Worker limit: that depends on a Cloudflare plan upgrade that hasn't happened yet. Revisit once
+ * the upgrade is confirmed and the 10-run wrangler tail test has been repeated.
+ */
+export const MAX_FRAGMENT_BYTES = 65_536;
 // Resolving "which identifier does this anchor call reference, and where is IT declared at the top
 // level" is plain lexical lookup, not the protected scope-boundary judgment — safe to do locally so a
 // schema declared far from its anchor (very common: define near the top, use in an export lower down)
@@ -30,6 +53,9 @@ function lineRange(node) {
         return null;
     return [node.loc.start.line - 1, node.loc.end.line - 1]; // 0-based, inclusive
 }
+function containsLine(range, line0) {
+    return line0 >= range[0] && line0 <= range[1];
+}
 function findEnclosingStatement(body, line0) {
     return body.find((stmt) => stmt.loc && stmt.loc.start.line - 1 <= line0 && line0 <= stmt.loc.end.line - 1);
 }
@@ -37,9 +63,51 @@ function findDeclarationStatement(body, name) {
     return body.find((stmt) => stmt.type === "VariableDeclaration" &&
         stmt.declarations.some((d) => d.id.type === "Identifier" && d.id.name === name));
 }
+/**
+ * Within `root`'s own line range, finds every function body that contains none of `protectedLines`
+ * (the real anchor match lines) and returns the set of its INNER line numbers (never the lines holding
+ * the opening/closing brace themselves, so the brace pair that keeps the surrounding statement valid is
+ * always kept). A function that DOES contain a protected line is left alone -- its content is exactly
+ * the point of sending this fragment -- but still descended into, in case it wraps a further, genuinely
+ * unrelated nested closure of its own.
+ */
+function findElidableInnerLines(file, rootRange, protectedLines) {
+    const elidable = new Set();
+    // @babel/traverse requires a Program/File root (not an arbitrary sub-statement) -- traverse the
+    // whole file and use rootRange to scope the search to just the one enclosing statement.
+    traverse(file, {
+        Function(path) {
+            const body = path.node.body;
+            if (body.type !== "BlockStatement")
+                return; // concise arrow body (`() => x`) -- nothing to strip
+            const bodyRange = lineRange(body);
+            if (!bodyRange)
+                return;
+            if (bodyRange[0] < rootRange[0] || bodyRange[1] > rootRange[1])
+                return; // outside root entirely
+            // Only the lines STRICTLY BETWEEN the braces count as "inside" this block for protection
+            // purposes -- the opening-brace line very often also carries the anchor itself (formbricks'
+            // `actionClient.inputSchema(X).action(async () => {` is one line), and that anchor is textually
+            // before the `{`, not inside the block it introduces. Using the full inclusive range here would
+            // make every such block look "protected" and defeat elision for exactly the common case this
+            // exists to handle.
+            const innerRange = [bodyRange[0] + 1, bodyRange[1] - 1];
+            if (protectedLines.some((line0) => containsLine(innerRange, line0)))
+                return; // keep + descend further
+            for (let i = innerRange[0]; i <= innerRange[1]; i++)
+                elidable.add(i);
+            path.skip(); // the whole block is already elided; no need to look inside it
+        },
+    });
+    return elidable;
+}
+function importedLocalNames(decl) {
+    return decl.specifiers.map((spec) => spec.local.name);
+}
 export function findZodCandidates(source, filePath, root) {
+    const none = { candidates: [], oversizedFiles: [] };
     if (!ZOD_IMPORT_RE.test(source))
-        return [];
+        return none;
     const lines = source.split("\n");
     const matchLineIndexes = [];
     lines.forEach((line, i) => {
@@ -47,12 +115,13 @@ export function findZodCandidates(source, filePath, root) {
             matchLineIndexes.push(i);
     });
     if (matchLineIndexes.length === 0)
-        return [];
+        return none;
     const ast = parseSource(source, filePath);
     if (!ast)
-        return [];
+        return none;
     const body = ast.program.body;
     const includedLines = new Set();
+    const excludedLines = new Set();
     const includeRange = (range) => {
         if (!range)
             return;
@@ -61,31 +130,53 @@ export function findZodCandidates(source, filePath, root) {
     };
     for (const directive of ast.program.directives)
         includeRange(lineRange(directive));
-    for (const stmt of body) {
-        if (stmt.type === "ImportDeclaration")
-            includeRange(lineRange(stmt));
-    }
+    const processedStatements = new Set();
     // One candidate per file, not per match — a single fragment covering every anchor-shaped
-    // statement plus the top-of-file imports, merged and deduped. Keeps this a single call per file
+    // statement plus the schemas it references, merged and deduped. Keeps this a single call per file
     // rather than one per match; the server already handles "multiple schemas in one fragment" naturally.
     for (const matchLine of matchLineIndexes) {
         const enclosing = findEnclosingStatement(body, matchLine);
         if (!enclosing)
             continue;
-        includeRange(lineRange(enclosing));
+        if (!processedStatements.has(enclosing)) {
+            processedStatements.add(enclosing);
+            const stmtRange = lineRange(enclosing);
+            if (stmtRange) {
+                includeRange(stmtRange);
+                for (const line0 of findElidableInnerLines(ast, stmtRange, matchLineIndexes))
+                    excludedLines.add(line0);
+            }
+        }
         const line = lines[matchLine];
         const referencedName = line.match(INPUT_SCHEMA_ARG_RE)?.[1] ?? line.match(PARSE_RECEIVER_RE)?.[1];
         if (!referencedName)
             continue;
         const declStmt = findDeclarationStatement(body, referencedName);
         if (declStmt)
-            includeRange(lineRange(declStmt));
+            includeRange(lineRange(declStmt)); // the schema itself -- never elided internally
     }
     if (includedLines.size === 0)
-        return [];
-    const sortedIndexes = [...includedLines].sort((a, b) => a - b);
+        return none;
+    // Only the imports actually referenced by what's being sent -- not every import in the file. A
+    // simple whole-word scan of the (not yet import-inclusive) fragment text, not real scope resolution:
+    // this can only ever over-include (a name that merely appears in a comment or string still pulls its
+    // import in), never under-include a genuinely used binding, which is the safe direction to err in.
+    const coreIndexes = [...includedLines].filter((i) => !excludedLines.has(i));
+    const coreText = coreIndexes.map((i) => lines[i]).join("\n");
+    for (const stmt of body) {
+        if (stmt.type !== "ImportDeclaration")
+            continue;
+        const referenced = importedLocalNames(stmt).some((name) => new RegExp(`\\b${name}\\b`).test(coreText));
+        if (referenced)
+            includeRange(lineRange(stmt));
+    }
+    const sortedIndexes = [...includedLines].filter((i) => !excludedLines.has(i)).sort((a, b) => a - b);
     const fragment = sortedIndexes.map((i) => lines[i]).join("\n");
     const fragmentLineToSourceLine = sortedIndexes.map((i) => i + 1);
     const relPath = relative(root, filePath);
-    return [{ id: relPath, codeFragment: redactSecrets(fragment), fragmentLineToSourceLine }];
+    const redacted = redactSecrets(fragment);
+    if (Buffer.byteLength(redacted, "utf8") > MAX_FRAGMENT_BYTES) {
+        return { candidates: [], oversizedFiles: [relPath] };
+    }
+    return { candidates: [{ id: relPath, codeFragment: redacted, fragmentLineToSourceLine }], oversizedFiles: [] };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "srclawyer",
-  "version": "0.4.1",
+  "version": "0.6.0",
   "description": "Static analysis engine that derives privacy policy documentation directly from source code",
   "license": "SEE LICENSE IN LICENSE",
   "type": "module",
@@ -24,7 +24,7 @@
     "postinstall.cjs"
   ],
   "scripts": {
-    "build": "tsc -p tsconfig.json",
+    "build": "tsc -p tsconfig.json && node scripts/copyGrammars.js",
     "prepare": "npm run build",
     "postinstall": "node postinstall.cjs",
     "dev": "tsx src/cli.ts",
@@ -41,6 +41,7 @@
     "fast-glob": "^3.3.2",
     "js-yaml": "^4.1.0",
     "typescript": "^5.6.2",
+    "web-tree-sitter": "^0.27.0",
     "yaml": "^2.5.1"
   },
   "devDependencies": {