src2sink 1.0.0__py3-none-any.whl

src2sink/constants.py

@@ -0,0 +1,44 @@
+"""Shared constants for metabase extractors (v1 + v2)."""
+
+from __future__ import annotations
+
+import re
+
+SKIP_DIRS = {
+    ".git", ".idea", ".vscode", ".gradle", ".mvn", ".pytest_cache",
+    "node_modules", "target", "build", "dist", "out", "bin",
+    "vendor", ".terraform", "__pycache__", ".cache", ".tox",
+    "coverage", ".nyc_output", ".next", ".nuxt", ".svelte-kit",
+    "tmp", "temp", ".DS_Store",
+    ".tmp_gitlab_archives",
+}
+
+SOURCE_EXTENSIONS = {
+    ".java": "java", ".kt": "kotlin", ".scala": "scala",
+    ".ts": "typescript", ".tsx": "tsx",
+    ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript",
+    ".py": "python", ".go": "go",
+}
+
+CONFIG_FILE_NAMES = {
+    "application.yml", "application.yaml", "application.properties",
+    "bootstrap.yml", "bootstrap.yaml", "bootstrap.properties",
+}
+
+MAX_FILE_BYTES = 1_500_000
+
+TEST_PATH_RX = re.compile(
+    r"(?:^|/)(?:"
+    r"test|tests|spec|specs|__tests__|fixtures?|examples?|samples?|"
+    r"sandbox|playground|local|demo|mock|mocks|stub|stubs|"
+    r"src/test|src/it|integration[-_]?test|"
+    r"qa[-_]?test|e2e"
+    r"|"
+    r"[a-z][a-zA-Z0-9]*Tests?"
+    r")(?:/|$)",
+    re.IGNORECASE,
+)
+
+WEAK_ALGOS = frozenset({
+    "MD5", "MD4", "SHA1", "SHA-1", "DES", "RC4", "RC2", "BLOWFISH",
+})

src2sink/curate_internal_libraries.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Auto-fill internal-library taint tables from cloned library source (stdlib)."""
+
+from __future__ import annotations
+
+import argparse
+import re
+from pathlib import Path
+from typing import Any
+
+from .aggregators.library_source_map import load_library_source_map
+from .library_taint_java import render_taint_table, scan_java_public_api
+from .repo_utils import _locate_library_source
+
+TBD_ROW = "| _TBD_ | _TBD_ | _TBD_ | _TBD_ | _TBD_ |\n"
+
+
+def _extract_coordinate(lib_text: str) -> str | None:
+    """Return the coordinate value from a library markdown table, or None."""
+    m = re.search(r"\|\s*Coordinate\s*\|\s*([^|]+)\s*\|", lib_text)
+    return m.group(1).strip() if m else None
+
+
+def _resolve_library_root(
+    repos_root: Path, coord: str, source_map: dict[str, Any] | None = None
+) -> Path | None:
+    """Resolve the on-disk source root for a library coordinate, or None.
+
+    Args:
+        repos_root: Root directory holding cloned repos.
+        coord: Library coordinate to locate.
+        source_map: Optional precomputed coordinate-to-source mapping.
+
+    Returns:
+        The library's source directory if it exists, else None.
+    """
+    source_rel = _locate_library_source(repos_root, coord, source_map=source_map)
+    if not source_rel:
+        return None
+    rel = source_rel.removeprefix("repos/").lstrip("/")
+    repo_root = repos_root / rel
+    return repo_root if repo_root.is_dir() else None
+
+
+def _replace_tbd_table(lib_text: str, table: str) -> str:
+    """Substitute the placeholder taint table in lib_text with the given table."""
+    if TBD_ROW in lib_text:
+        return lib_text.replace(TBD_ROW, _table_body(table), count=1)
+    return re.sub(
+        r"(## Public-API taint table \(hand-curated\)\n\n"
+        r"(?:Each row.*?\n\n)?)"
+        r"\| Method signature \|.*?\| _TBD_ \| _TBD_ \| _TBD_ \| _TBD_ \| _TBD_ \|\n",
+        r"\1" + table + "\n",
+        lib_text,
+        count=1,
+        flags=re.DOTALL,
+    )
+
+
+def _table_body(table: str) -> str:
+    """Return the data rows of a markdown table, dropping the header and separator."""
+    body_lines = table.strip().splitlines()
+    if len(body_lines) > 2:
+        return "\n".join(body_lines[2:]) + "\n"
+    return table
+
+
+def curate_library_file(
+    lib_path: Path,
+    repos_root: Path,
+    coord: str,
+    source_map: dict[str, Any] | None = None,
+) -> bool:
+    """Fill a library file's placeholder taint table from scanned source.
+
+    Args:
+        lib_path: Path to the internal-library markdown file.
+        repos_root: Root directory holding cloned repos.
+        coord: Library coordinate used to locate the source.
+        source_map: Optional precomputed coordinate-to-source mapping.
+
+    Returns:
+        True if the file was updated, False otherwise.
+    """
+    text = lib_path.read_text(encoding="utf-8")
+    if "_TBD_" not in text:
+        return False
+    repo_root = _resolve_library_root(repos_root, coord, source_map=source_map)
+    if not repo_root:
+        return False
+    rows = scan_java_public_api(repo_root)
+    if not rows:
+        return False
+    table = render_taint_table(rows)
+    new_text = _replace_tbd_table(text, table)
+    if new_text == text:
+        return False
+    lib_path.write_text(new_text, encoding="utf-8")
+    return True
+
+
+def main() -> int:
+    """CLI entry point: curate placeholder taint tables across internal libraries."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repos-root", required=True)
+    parser.add_argument("--metabase-root", required=True)
+    parser.add_argument("--limit", type=int, default=30)
+    args = parser.parse_args()
+
+    repos_root = Path(args.repos_root).resolve()
+    metabase_root = Path(args.metabase_root).resolve()
+    lib_dir = metabase_root / "internal-libraries"
+    lib_dir.mkdir(parents=True, exist_ok=True)
+    source_map = load_library_source_map(metabase_root / "library-source-map.json")
+    updated = 0
+    for path in sorted(lib_dir.glob("*.md")):
+        if path.name == "INDEX.md":
+            continue
+        text = path.read_text(encoding="utf-8")
+        if "_TBD_" not in text:
+            continue
+        coord = _extract_coordinate(text)
+        if not coord:
+            continue
+        if curate_library_file(path, repos_root, coord, source_map=source_map):
+            updated += 1
+            print(f"  curated {path.name}")
+        if updated >= args.limit:
+            break
+    print(f"Updated {updated} library files")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src2sink/extractors/__init__.py

@@ -0,0 +1,17 @@
+"""Per-language tree-sitter extractors (Phase 1+)."""
+
+from .base import (
+    load_language,
+    make_parser,
+    parse_file,
+    parse_source,
+    supported_languages,
+)
+
+__all__ = [
+    "load_language",
+    "make_parser",
+    "parse_file",
+    "parse_source",
+    "supported_languages",
+]

src2sink/extractors/ast_walk.py

@@ -0,0 +1,131 @@
+"""Tree-sitter AST walking helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from tree_sitter import Node
+
+
+def node_text(source: bytes, node: Node) -> str:
+    """Decode the source bytes spanned by ``node`` to text (errors replaced)."""
+    return source[node.start_byte : node.end_byte].decode("utf-8", errors="replace")
+
+
+def line_number(source: bytes, node: Node) -> int:
+    """Return the 1-based line number where ``node`` starts."""
+    return node.start_point[0] + 1
+
+
+def walk(node: Node) -> Iterator[Node]:
+    """Yield ``node`` and every descendant via an iterative pre-order traversal."""
+    stack = [node]
+    while stack:
+        current = stack.pop()
+        yield current
+        for child in reversed(current.children):
+            stack.append(child)
+
+
+def call_name_java_kotlin(source: bytes, node: Node) -> str | None:
+    """Return the invoked method name for a Java/Kotlin call node, or None."""
+    if node.type != "method_invocation":
+        return None
+    name_node = node.child_by_field_name("name")
+    if name_node is not None:
+        return node_text(source, name_node)
+    for child in reversed(node.children):
+        if child.type in ("identifier", "field_identifier"):
+            return node_text(source, child)
+    return None
+
+
+def call_name_python(source: bytes, node: Node) -> str | None:
+    """Return the called function/attribute name for a Python call node, or None."""
+    if node.type != "call":
+        return None
+    fn = node.child_by_field_name("function")
+    if fn is None:
+        return None
+    if fn.type == "identifier":
+        return node_text(source, fn)
+    if fn.type == "attribute":
+        attr = fn.child_by_field_name("attribute")
+        if attr is not None:
+            return node_text(source, attr)
+    return None
+
+
+def call_name_js_go(source: bytes, node: Node, language: str) -> str | None:
+    """Return the called function/member name for a JS/TS/Go call node, or None."""
+    if language in ("javascript", "typescript", "tsx"):
+        if node.type != "call_expression":
+            return None
+        fn = node.child_by_field_name("function")
+        if fn is None:
+            return None
+        if fn.type == "identifier":
+            return node_text(source, fn)
+        if fn.type == "member_expression":
+            prop = fn.child_by_field_name("property")
+            if prop is not None:
+                return node_text(source, prop)
+    if language == "go":
+        if node.type != "call_expression":
+            return None
+        fn = node.child_by_field_name("function")
+        if fn is None:
+            return None
+        if fn.type == "selector_expression":
+            field = fn.child_by_field_name("field")
+            if field is not None:
+                return node_text(source, field)
+        if fn.type == "identifier":
+            return node_text(source, fn)
+    return None
+
+
+def extract_call_name(source: bytes, node: Node, language: str) -> str | None:
+    """Dispatch to the language-specific call-name extractor for ``node``."""
+    if language in ("java", "kotlin"):
+        return call_name_java_kotlin(source, node)
+    if language == "python":
+        return call_name_python(source, node)
+    return call_name_js_go(source, node, language)
+
+
+CALL_NODE_TYPES: dict[str, frozenset[str]] = {
+    "java": frozenset({"method_invocation"}),
+    "kotlin": frozenset({"call_expression", "navigation_expression"}),
+    "python": frozenset({"call"}),
+    "javascript": frozenset({"call_expression"}),
+    "typescript": frozenset({"call_expression"}),
+    "tsx": frozenset({"call_expression"}),
+    "go": frozenset({"call_expression"}),
+}
+
+
+def iter_calls(
+    source: bytes,
+    root: Node,
+    language: str,
+    *,
+    types: frozenset[str] | None = None,
+) -> Iterator[tuple[Node, str]]:
+    """Yield (call node, call name) for every call-expression node in the subtree.
+
+    Args:
+        source: Raw source bytes (untrusted scanned code) backing the tree.
+        root: Subtree root to walk.
+        language: Language id used to pick call node types and name extraction.
+        types: Optional override of node types treated as calls.
+    """
+    types = types or CALL_NODE_TYPES.get(language, frozenset())
+    for node in walk(root):
+        if node.type not in types:
+            continue
+        name = extract_call_name(source, node, language)
+        if name:
+            yield node, name

src2sink/extractors/base.py

@@ -0,0 +1,59 @@
+"""Tree-sitter parser loading and shared extractor utilities."""
+
+from __future__ import annotations
+
+import importlib
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from tree_sitter import Language, Parser, Tree
+
+_LANGUAGE_MODULES: dict[str, tuple[str, str]] = {
+    "java": ("tree_sitter_java", "language"),
+    "python": ("tree_sitter_python", "language"),
+    "javascript": ("tree_sitter_javascript", "language"),
+    "go": ("tree_sitter_go", "language"),
+    "kotlin": ("tree_sitter_kotlin", "language"),
+    "typescript": ("tree_sitter_typescript", "language_typescript"),
+    "tsx": ("tree_sitter_typescript", "language_tsx"),
+}
+
+
+def load_language(language_id: str) -> Language:
+    """Load and return the tree-sitter Language for ``language_id``.
+
+    Raises KeyError for an unsupported language id.
+    """
+    if language_id not in _LANGUAGE_MODULES:
+        raise KeyError(f"unsupported language: {language_id}")
+    module_name, attr = _LANGUAGE_MODULES[language_id]
+    mod = importlib.import_module(module_name)
+    factory = getattr(mod, attr)
+    from tree_sitter import Language
+
+    return Language(factory())
+
+
+def make_parser(language_id: str) -> Parser:
+    """Return a tree-sitter Parser configured for ``language_id``."""
+    from tree_sitter import Parser
+
+    return Parser(load_language(language_id))
+
+
+def parse_source(language_id: str, source: str | bytes) -> Tree:
+    """Parse untrusted ``source`` into a tree-sitter Tree; the code is never executed."""
+    if isinstance(source, str):
+        source = source.encode("utf-8")
+    return make_parser(language_id).parse(source)
+
+
+def parse_file(language_id: str, path: Path) -> Tree:
+    """Read ``path`` and parse its (untrusted) contents into a tree-sitter Tree."""
+    return parse_source(language_id, path.read_bytes())
+
+
+def supported_languages() -> frozenset[str]:
+    """Return the set of language ids with a registered tree-sitter grammar."""
+    return frozenset(_LANGUAGE_MODULES)

src2sink/extractors/config.py

@@ -0,0 +1,224 @@
+"""Config-file extractor — JDBC/NoSQL URLs, security keys, crypto-from-config."""
+
+from __future__ import annotations
+
+import re
+
+from .node_factory import make_node
+from ..schema import FlowEdge, FlowNode
+
+# v1 security-sensitive keys (properties / YAML flattened keys)
+# Flat properties / env style (v1 parity)
+SECURITY_CONFIG_KEYS = re.compile(
+    r"^\s*(security\.enabled|micronaut\.security\.enabled|"
+    r"enable\.serverauth|csrf\.disabled|csrf\.enabled|"
+    r"server\.ssl\.enabled|tls\.enabled|verify[-_.]?ssl|"
+    r"insecure[_-]?skip[_-]?verify|"
+    r"cors\.allowed[\.-]origins|cors\.allow[\.-]?credentials|"
+    r"management\.endpoints\.web\.exposure\.include|"
+    r"spring\.security\.user\.password|"
+    r"jwt\.secret|jwt\.signing[\.-]?key|"
+    r"oauth2?\.client[\.-]?secret|"
+    r"actuator\.endpoints|"
+    r"debug\s*[:=]\s*true)",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+# YAML / Helm nested keys (same semantics as dotted properties)
+YAML_SECURITY_KEYS = re.compile(
+    r"^\s*(?:"
+    r"password|secret|signing[-_]?algorithm|client[-_]?secret|"
+    r"csrf\.(?:enabled|disabled)|"
+    r"allowed[-_]?origins|allow[-_]?credentials"
+    r")\s*:",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+JDBC_URL_RX = re.compile(
+    r"jdbc:(?:postgresql|mysql|mariadb|oracle|sqlserver|h2|sqlite)[^\"'\s]{8,}",
+    re.IGNORECASE,
+)
+
+MONGODB_URI_RX = re.compile(
+    r"mongodb(?:\+srv)?://[^\"'\s]{8,}",
+    re.IGNORECASE,
+)
+
+REDIS_URL_RX = re.compile(
+    r"rediss?://[^\"'\s]{4,}",
+    re.IGNORECASE,
+)
+
+S3_BUCKET_RX = re.compile(
+    r"(?:s3://([a-z0-9.\-]{3,63})|bucket[-_.]?name\s*[=:]\s*['\"]?([a-z0-9.\-]{3,63}))",
+    re.IGNORECASE,
+)
+
+CRYPTO_CONFIG_RX: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r"server\.ssl\.ciphers?\s*[=:]", re.I), "tls-cipher-suite"),
+    (re.compile(r"encryption\.algorithm\s*[=:]", re.I), "encryption-algorithm"),
+    (re.compile(r"crypto\.algorithm\s*[=:]", re.I), "encryption-algorithm"),
+    (re.compile(r"jwt\.signing[-_.]?algorithm\s*[=:]", re.I), "jwt-algorithm"),
+    (re.compile(r"spring\.ssl\.bundle", re.I), "tls-bundle"),
+    # YAML / Helm nested keys
+    (re.compile(r"^\s*ciphers\s*:", re.I | re.M), "tls-cipher-suite"),
+    (re.compile(r"^\s*signing[-_]?algorithm\s*:", re.I | re.M), "jwt-algorithm"),
+]
+
+SECRET_REF_RX: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r"aws\.secretsmanager|secretsmanager:", re.I), "aws-secrets-manager"),
+    (re.compile(r"vault[:\.]|hashicorp\.vault", re.I), "hashicorp-vault"),
+    (re.compile(r"azure\.keyvault|keyvault\.vault", re.I), "azure-key-vault"),
+    (re.compile(r"secretmanager\.googleapis|gcp\.secret", re.I), "gcp-secret-manager"),
+    (re.compile(r"kms\.key|aws\.kms|alias/", re.I), "aws-kms"),
+]
+
+DATASOURCE_KEY_RX = re.compile(
+    r"(?:spring\.datasource\.url|datasource\.url|"
+    r"quarkus\.datasource\.jdbc\.url|"
+    r"micronaut\.datasources\.[^.\s]+\.url)"
+    r"\s*[=:]\s*['\"]?([^\"'\n]+)",
+    re.IGNORECASE,
+)
+
+
+def _line_of(text: str, pos: int) -> int:
+    """Return the 1-based line number of byte offset ``pos`` in ``text``."""
+    return text.count("\n", 0, pos) + 1
+
+
+def _line_slice(text: str, pos: int, end: int) -> str:
+    """Return the trimmed full source line spanning ``pos``..``end`` (max 200 chars)."""
+    line_end = text.find("\n", end)
+    if line_end < 0:
+        line_end = len(text)
+    line_start = text.rfind("\n", 0, pos) + 1
+    return text[line_start:line_end].strip()[:200]
+
+
+def extract_from_config(
+    *,
+    repo_id: str,
+    rel_path: str,
+    source: str,
+) -> tuple[list[FlowNode], list[FlowEdge]]:
+    """Extract config-file flow nodes (security keys, datastore URLs, crypto, secrets).
+
+    Scans untrusted config text via regex; matches are only recorded, never evaluated.
+    Returns the accumulated nodes and (currently empty) edges.
+    """
+    nodes: list[FlowNode] = []
+    edges: list[FlowEdge] = []
+
+    for m in SECURITY_CONFIG_KEYS.finditer(source):
+        key = m.group(1)
+        nodes.append(make_node(
+            repo=repo_id,
+            file=rel_path,
+            line=_line_of(source, m.start()),
+            language="config",
+            kind="propagator",
+            family="config-security",
+            detail={"key": key, "value_line": _line_slice(source, m.start(), m.end())},
+            confidence="high",
+        ))
+
+    for m in YAML_SECURITY_KEYS.finditer(source):
+        key = m.group(0).strip().split(":")[0].strip()
+        nodes.append(make_node(
+            repo=repo_id,
+            file=rel_path,
+            line=_line_of(source, m.start()),
+            language="config",
+            kind="propagator",
+            family="config-security",
+            detail={"key": key, "value_line": _line_slice(source, m.start(), m.end())},
+            confidence="medium",
+        ))
+
+    for m in DATASOURCE_KEY_RX.finditer(source):
+        url = m.group(1).strip()
+        nodes.append(make_node(
+            repo=repo_id,
+            file=rel_path,
+            line=_line_of(source, m.start()),
+            language="config",
+            kind="store",
+            family="data-store",
+            detail={"vendor": "jdbc", "url": url[:200], "source": "datasource-key"},
+            confidence="high",
+        ))
+
+    for pat, vendor in (
+        (JDBC_URL_RX, "jdbc"),
+        (MONGODB_URI_RX, "mongodb"),
+        (REDIS_URL_RX, "redis"),
+    ):
+        for m in pat.finditer(source):
+            nodes.append(make_node(
+                repo=repo_id,
+                file=rel_path,
+                line=_line_of(source, m.start()),
+                language="config",
+                kind="store",
+                family="data-store",
+                detail={"vendor": vendor, "url": m.group(0)[:200], "source": "uri-pattern"},
+                confidence="high",
+            ))
+
+    for m in S3_BUCKET_RX.finditer(source):
+        bucket = m.group(1) or m.group(2) or ""
+        if bucket:
+            nodes.append(make_node(
+                repo=repo_id,
+                file=rel_path,
+                line=_line_of(source, m.start()),
+                language="config",
+                kind="store",
+                family="data-store",
+                detail={"vendor": "s3", "bucket": bucket, "source": "bucket-ref"},
+                confidence="medium",
+            ))
+
+    for pat, sub in CRYPTO_CONFIG_RX:
+        for m in pat.finditer(source):
+            nodes.append(make_node(
+                repo=repo_id,
+                file=rel_path,
+                line=_line_of(source, m.start()),
+                language="config",
+                kind="propagator",
+                family="crypto-config",
+                detail={"subkind": sub, "line": _line_slice(source, m.start(), m.end())},
+                confidence="high",
+            ))
+
+    for pat, provider in SECRET_REF_RX:
+        for m in pat.finditer(source):
+            nodes.append(make_node(
+                repo=repo_id,
+                file=rel_path,
+                line=_line_of(source, m.start()),
+                language="config",
+                kind="propagator",
+                family="crypto-key-source",
+                detail={"provider": provider, "from": "config", "raw": m.group(0)[:80]},
+                confidence="medium",
+            ))
+
+    return nodes, edges
+
+
+def is_config_path(path_name: str, suffix: str) -> bool:
+    """True when a file name/suffix identifies it as a config file for this extractor."""
+    from ..constants import CONFIG_FILE_NAMES
+
+    if path_name in CONFIG_FILE_NAMES:
+        return True
+    if path_name.startswith("application-") and suffix in {".yml", ".yaml", ".properties"}:
+        return True
+    if suffix in {".properties", ".yml", ".yaml", ".env"}:
+        return True
+    if path_name in {"values.yaml", "values.yml"} or path_name.endswith(".values.yaml"):
+        return True
+    return False

src2sink/extractors/file_context.py

@@ -0,0 +1,35 @@
+"""Per-file extraction state passed between regex and tree-sitter passes."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ..schema import FlowEdge, FlowNode
+
+
+@dataclass
+class FileExtractionContext:
+    """Mutable accumulator while scanning one source file."""
+
+    repo_id: str
+    rel_path: str
+    language: str
+    source: str
+    nodes: list[FlowNode] = field(default_factory=list)
+    edges: list[FlowEdge] = field(default_factory=list)
+    # Cross-pass links for raw-code-payload (endpoint + sql field + execution sink).
+    http_sources: list[FlowNode] = field(default_factory=list)
+    sql_execution_sinks: list[FlowNode] = field(default_factory=list)
+    raw_sql_field_lines: list[int] = field(default_factory=list)
+
+    def line_number(self, pos: int) -> int:
+        """Return the 1-based line number of byte offset ``pos`` in the source."""
+        return self.source.count("\n", 0, pos) + 1
+
+    def line_text_at(self, pos: int) -> str:
+        """Full source line containing byte offset ``pos``."""
+        start = self.source.rfind("\n", 0, pos) + 1
+        end = self.source.find("\n", pos)
+        if end == -1:
+            end = len(self.source)
+        return self.source[start:end]

src2sink/extractors/go.py

@@ -0,0 +1,3 @@
+"""Go extractor — Phase 1."""
+
+from __future__ import annotations