codebread 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codebread/__init__.py +3 -0
- codebread/analyzer.py +84 -0
- codebread/classifier.py +102 -0
- codebread/cli.py +106 -0
- codebread/connections.py +407 -0
- codebread/diff.py +123 -0
- codebread/export.py +45 -0
- codebread/languages.py +110 -0
- codebread/models.py +119 -0
- codebread/parsers/__init__.py +53 -0
- codebread/parsers/generic_parser.py +288 -0
- codebread/parsers/javascript_parser.py +371 -0
- codebread/parsers/python_parser.py +291 -0
- codebread/scanner.py +206 -0
- codebread/server.py +86 -0
- codebread/web/app.js +1716 -0
- codebread/web/index.html +188 -0
- codebread/web/style.css +601 -0
- codebread-1.0.0.dist-info/METADATA +254 -0
- codebread-1.0.0.dist-info/RECORD +24 -0
- codebread-1.0.0.dist-info/WHEEL +5 -0
- codebread-1.0.0.dist-info/entry_points.txt +2 -0
- codebread-1.0.0.dist-info/licenses/LICENSE +21 -0
- codebread-1.0.0.dist-info/top_level.txt +1 -0
codebread/languages.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Language detection by extension + light content sniffing."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
# extension -> language id
|
|
7
|
+
EXT_MAP = {
|
|
8
|
+
".py": "python", ".pyw": "python",
|
|
9
|
+
".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
|
|
10
|
+
".ts": "typescript", ".tsx": "typescript", ".mts": "typescript", ".cts": "typescript",
|
|
11
|
+
".vue": "vue", ".svelte": "svelte",
|
|
12
|
+
".java": "java",
|
|
13
|
+
".go": "go",
|
|
14
|
+
".php": "php",
|
|
15
|
+
".rb": "ruby", ".rake": "ruby",
|
|
16
|
+
".cs": "csharp",
|
|
17
|
+
".sql": "sql",
|
|
18
|
+
".html": "html", ".htm": "html",
|
|
19
|
+
".css": "css", ".scss": "css", ".sass": "css", ".less": "css",
|
|
20
|
+
".json": "config", ".yaml": "config", ".yml": "config", ".toml": "config",
|
|
21
|
+
".ini": "config", ".cfg": "config", ".conf": "config", ".env": "config",
|
|
22
|
+
".properties": "config", ".xml": "config",
|
|
23
|
+
".md": "markdown", ".rst": "markdown", ".txt": "text",
|
|
24
|
+
".sh": "shell", ".bash": "shell", ".ps1": "shell", ".bat": "shell", ".cmd": "shell",
|
|
25
|
+
# known-but-unparsed languages (surface as "unsupported", never silently skip)
|
|
26
|
+
".rs": "rust", ".kt": "kotlin", ".kts": "kotlin", ".swift": "swift",
|
|
27
|
+
".c": "c", ".h": "c", ".cpp": "cpp", ".cc": "cpp", ".hpp": "cpp",
|
|
28
|
+
".scala": "scala", ".ex": "elixir", ".exs": "elixir", ".erl": "erlang",
|
|
29
|
+
".lua": "lua", ".r": "r", ".pl": "perl", ".dart": "dart", ".zig": "zig",
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
CONFIG_BASENAMES = {
|
|
33
|
+
".env", ".env.local", ".env.development", ".env.production", ".env.example",
|
|
34
|
+
"dockerfile", "docker-compose.yml", "docker-compose.yaml", "makefile",
|
|
35
|
+
"settings.py", "config.py", "config.js", "config.ts", ".babelrc", ".eslintrc",
|
|
36
|
+
"tsconfig.json", "package.json", "pyproject.toml", "setup.py", "setup.cfg",
|
|
37
|
+
"webpack.config.js", "vite.config.js", "vite.config.ts", "next.config.js",
|
|
38
|
+
"tailwind.config.js", "requirements.txt", "gemfile", "pom.xml", "build.gradle",
|
|
39
|
+
"go.mod", "cargo.toml", "composer.json", "appsettings.json", "web.config",
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
# languages CodeBread can actually parse
|
|
43
|
+
PARSED = {"python", "javascript", "typescript", "vue", "svelte",
|
|
44
|
+
"java", "go", "php", "ruby", "csharp", "sql"}
|
|
45
|
+
|
|
46
|
+
# languages that exist but have no parser -> shown as unsupported warnings
|
|
47
|
+
KNOWN_UNPARSED = {"rust", "kotlin", "swift", "c", "cpp", "scala", "elixir",
|
|
48
|
+
"erlang", "lua", "r", "perl", "dart", "zig", "shell"}
|
|
49
|
+
|
|
50
|
+
BINARY_EXTS = {
|
|
51
|
+
".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".svg", ".avif",
|
|
52
|
+
".woff", ".woff2", ".ttf", ".eot", ".otf",
|
|
53
|
+
".mp3", ".mp4", ".wav", ".ogg", ".webm", ".mov", ".avi",
|
|
54
|
+
".zip", ".gz", ".tar", ".rar", ".7z", ".bz2",
|
|
55
|
+
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
|
|
56
|
+
".exe", ".dll", ".so", ".dylib", ".bin", ".o", ".a", ".class", ".jar",
|
|
57
|
+
".pyc", ".pyo", ".pyd", ".db", ".sqlite", ".sqlite3", ".DS_Store",
|
|
58
|
+
".lock", ".map", ".wasm",
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
LOCKFILES = {
|
|
62
|
+
"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock",
|
|
63
|
+
"pipfile.lock", "composer.lock", "cargo.lock", "gemfile.lock", "go.sum",
|
|
64
|
+
"bun.lockb",
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def detect_language(path: str) -> str:
|
|
69
|
+
"""Best-effort language id for a file path (with shebang sniffing)."""
|
|
70
|
+
base = os.path.basename(path).lower()
|
|
71
|
+
if base in LOCKFILES:
|
|
72
|
+
return "lockfile"
|
|
73
|
+
if base in CONFIG_BASENAMES or base.startswith(".env"):
|
|
74
|
+
# config files keep their real language when parseable (settings.py etc.)
|
|
75
|
+
ext = os.path.splitext(base)[1]
|
|
76
|
+
lang = EXT_MAP.get(ext, "config")
|
|
77
|
+
return lang if lang in PARSED else "config"
|
|
78
|
+
ext = os.path.splitext(base)[1].lower()
|
|
79
|
+
if ext in EXT_MAP:
|
|
80
|
+
return EXT_MAP[ext]
|
|
81
|
+
if not ext:
|
|
82
|
+
# sniff a shebang
|
|
83
|
+
try:
|
|
84
|
+
with open(path, "rb") as f:
|
|
85
|
+
head = f.read(160)
|
|
86
|
+
if head.startswith(b"#!"):
|
|
87
|
+
line = head.splitlines()[0].decode("utf-8", "ignore")
|
|
88
|
+
if "python" in line:
|
|
89
|
+
return "python"
|
|
90
|
+
if "node" in line:
|
|
91
|
+
return "javascript"
|
|
92
|
+
if "ruby" in line:
|
|
93
|
+
return "ruby"
|
|
94
|
+
if "bash" in line or "sh" in line:
|
|
95
|
+
return "shell"
|
|
96
|
+
except OSError:
|
|
97
|
+
pass
|
|
98
|
+
return "unknown"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def looks_binary(path: str) -> bool:
|
|
102
|
+
ext = os.path.splitext(path)[1].lower()
|
|
103
|
+
if ext in BINARY_EXTS:
|
|
104
|
+
return True
|
|
105
|
+
try:
|
|
106
|
+
with open(path, "rb") as f:
|
|
107
|
+
chunk = f.read(2048)
|
|
108
|
+
return b"\x00" in chunk
|
|
109
|
+
except OSError:
|
|
110
|
+
return True
|
codebread/models.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Shared data structures for scan results."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass, field, asdict
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class ApiCall:
|
|
10
|
+
"""An outgoing HTTP call found in source (fetch/axios/requests/...)."""
|
|
11
|
+
method: str # GET/POST/... or ANY
|
|
12
|
+
url: str # raw url/path as written
|
|
13
|
+
line: int = 0
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class Route:
|
|
18
|
+
"""A server route definition (@app.route, app.get, @GetMapping, ...)."""
|
|
19
|
+
method: str # GET/POST/... or ANY
|
|
20
|
+
path: str
|
|
21
|
+
line: int = 0
|
|
22
|
+
handler: str = "" # handler function name, if separate from the decorated fn
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class DbRef:
|
|
27
|
+
"""A database touch: ORM model usage or raw SQL."""
|
|
28
|
+
table: str # table/collection/model name as written
|
|
29
|
+
op: str = "query" # query/insert/update/delete/define
|
|
30
|
+
via: str = "orm" # orm | sql
|
|
31
|
+
line: int = 0
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class FunctionInfo:
|
|
36
|
+
name: str
|
|
37
|
+
params: List[str] = field(default_factory=list)
|
|
38
|
+
returns: str = ""
|
|
39
|
+
line: int = 0
|
|
40
|
+
end_line: int = 0
|
|
41
|
+
doc: str = ""
|
|
42
|
+
description: str = ""
|
|
43
|
+
calls: List[str] = field(default_factory=list) # names of callees
|
|
44
|
+
api_calls: List[ApiCall] = field(default_factory=list)
|
|
45
|
+
routes: List[Route] = field(default_factory=list) # routes this fn handles
|
|
46
|
+
db_refs: List[DbRef] = field(default_factory=list)
|
|
47
|
+
parent_class: str = ""
|
|
48
|
+
kind: str = "function" # function | method
|
|
49
|
+
index: int = 0 # "Function 1", "Function 2" ... per file
|
|
50
|
+
code: str = "" # source snippet (capped), for the UI viewer
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class TableInfo:
|
|
55
|
+
"""A DB table/collection/model definition."""
|
|
56
|
+
name: str # table name
|
|
57
|
+
model: str = "" # ORM class name, if any
|
|
58
|
+
fields: List[str] = field(default_factory=list)
|
|
59
|
+
line: int = 0
|
|
60
|
+
source: str = "" # file that defines it
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class FileInfo:
|
|
65
|
+
path: str # relative, forward slashes
|
|
66
|
+
language: str = "unknown"
|
|
67
|
+
layer: str = "unknown" # frontend | backend | database | config | unknown
|
|
68
|
+
parsed: bool = False
|
|
69
|
+
loc: int = 0
|
|
70
|
+
imports: List[str] = field(default_factory=list)
|
|
71
|
+
calls: List[str] = field(default_factory=list) # top-level calls
|
|
72
|
+
links: List[str] = field(default_factory=list) # page navigations
|
|
73
|
+
functions: List[FunctionInfo] = field(default_factory=list)
|
|
74
|
+
tables: List[TableInfo] = field(default_factory=list)
|
|
75
|
+
api_calls: List[ApiCall] = field(default_factory=list) # top-level calls
|
|
76
|
+
routes: List[Route] = field(default_factory=list) # top-level route defs
|
|
77
|
+
db_refs: List[DbRef] = field(default_factory=list) # top-level db refs
|
|
78
|
+
warnings: List[str] = field(default_factory=list)
|
|
79
|
+
db_config: List[str] = field(default_factory=list) # masked config lines
|
|
80
|
+
source: str = "" # full text (capped)
|
|
81
|
+
|
|
82
|
+
def to_dict(self):
|
|
83
|
+
return asdict(self)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def humanize(name: str) -> str:
|
|
87
|
+
"""Turn get_user_by_id / getUserById into 'Get user by id'."""
|
|
88
|
+
import re
|
|
89
|
+
s = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", name)
|
|
90
|
+
s = s.replace("_", " ").replace("-", " ").strip()
|
|
91
|
+
s = re.sub(r"\s+", " ", s)
|
|
92
|
+
return (s[:1].upper() + s[1:]) if s else name
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def describe(fn: FunctionInfo, body_hint: str = "") -> str:
|
|
96
|
+
"""Auto-description: docstring first line, else heuristic summary."""
|
|
97
|
+
if fn.doc:
|
|
98
|
+
first = fn.doc.strip().splitlines()[0].strip()
|
|
99
|
+
if first:
|
|
100
|
+
return first
|
|
101
|
+
hints = []
|
|
102
|
+
if fn.routes:
|
|
103
|
+
r = fn.routes[0]
|
|
104
|
+
hints.append(f"handles {r.method} {r.path}")
|
|
105
|
+
if fn.api_calls:
|
|
106
|
+
hints.append("calls an API")
|
|
107
|
+
if fn.db_refs:
|
|
108
|
+
tables = sorted({d.table for d in fn.db_refs})
|
|
109
|
+
hints.append("queries " + ", ".join(tables[:3]))
|
|
110
|
+
low = body_hint.lower()
|
|
111
|
+
if not hints:
|
|
112
|
+
if "open(" in low or "readfile" in low or "writefile" in low:
|
|
113
|
+
hints.append("reads/writes files")
|
|
114
|
+
elif "render" in low or "jsx" in low or "createelement" in low:
|
|
115
|
+
hints.append("renders UI")
|
|
116
|
+
base = humanize(fn.name)
|
|
117
|
+
if hints:
|
|
118
|
+
return f"{base} — {'; '.join(hints)}."
|
|
119
|
+
return base + "."
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Parser dispatch: pick the right extractor for a file's language."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
from ..models import FileInfo
|
|
7
|
+
from ..languages import PARSED, KNOWN_UNPARSED
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def parse_file(rel_path: str, text: str, language: str) -> FileInfo:
|
|
11
|
+
"""Parse `text` and return a populated FileInfo. Never raises."""
|
|
12
|
+
info = FileInfo(path=rel_path, language=language)
|
|
13
|
+
info.loc = text.count("\n") + 1 if text else 0
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
if language == "python":
|
|
17
|
+
from .python_parser import parse_python
|
|
18
|
+
parse_python(info, text)
|
|
19
|
+
elif language in ("javascript", "typescript", "vue", "svelte"):
|
|
20
|
+
from .javascript_parser import parse_javascript
|
|
21
|
+
parse_javascript(info, text, language)
|
|
22
|
+
elif language in ("java", "go", "php", "ruby", "csharp"):
|
|
23
|
+
from .generic_parser import parse_generic
|
|
24
|
+
parse_generic(info, text, language)
|
|
25
|
+
elif language == "sql":
|
|
26
|
+
from .generic_parser import parse_sql
|
|
27
|
+
parse_sql(info, text)
|
|
28
|
+
elif language == "config":
|
|
29
|
+
from .generic_parser import parse_config
|
|
30
|
+
parse_config(info, text)
|
|
31
|
+
elif language in KNOWN_UNPARSED:
|
|
32
|
+
info.warnings.append(
|
|
33
|
+
f"Unsupported: {language} — parsing skipped.")
|
|
34
|
+
# html/css/markdown/text/unknown: listed in the tree, nothing to extract
|
|
35
|
+
info.parsed = language in PARSED or language == "config"
|
|
36
|
+
except Exception as exc: # a parser bug must never kill the scan
|
|
37
|
+
info.warnings.append(f"Parser error ({type(exc).__name__}): {exc}")
|
|
38
|
+
info.parsed = False
|
|
39
|
+
|
|
40
|
+
# assign "Function 1", "Function 2", ... numbering per file,
|
|
41
|
+
# and attach a source snippet for the UI's expandable code viewer
|
|
42
|
+
lines = text.splitlines()
|
|
43
|
+
for i, fn in enumerate(info.functions, 1):
|
|
44
|
+
fn.index = i
|
|
45
|
+
if 0 < fn.line <= len(lines):
|
|
46
|
+
end = max(fn.line, min(fn.end_line or fn.line, len(lines)))
|
|
47
|
+
snippet = lines[fn.line - 1:min(end, fn.line - 1 + 120)]
|
|
48
|
+
while snippet and not snippet[-1].strip():
|
|
49
|
+
snippet.pop()
|
|
50
|
+
fn.code = "\n".join(snippet)
|
|
51
|
+
if end - fn.line + 1 > 120:
|
|
52
|
+
fn.code += "\n… (truncated)"
|
|
53
|
+
return info
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""Regex-based extractors for Java, Go, PHP, Ruby, C#, plus SQL schema files
|
|
2
|
+
and config files (with credential masking). Less precise than a real parser,
|
|
3
|
+
but keeps these languages visible in the graph instead of silently skipped.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from typing import Dict, List, Pattern
|
|
9
|
+
|
|
10
|
+
from ..models import (ApiCall, DbRef, FileInfo, FunctionInfo, Route,
|
|
11
|
+
TableInfo, describe)
|
|
12
|
+
|
|
13
|
+
KEYWORDS = {
|
|
14
|
+
"if", "for", "while", "switch", "catch", "return", "new", "else", "do",
|
|
15
|
+
"try", "throw", "case", "using", "lock", "foreach", "select", "func",
|
|
16
|
+
"defer", "go", "range", "elsif", "unless", "until", "when", "match",
|
|
17
|
+
"sizeof", "typeof", "yield", "puts", "print", "println", "printf",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
SQL_RE = re.compile(
|
|
21
|
+
r"\b(SELECT|INSERT|UPDATE|DELETE)\b[\s\S]{0,200}?"
|
|
22
|
+
r"\b(?:FROM|INTO|UPDATE|JOIN)\s+[`\"\[]?([A-Za-z_][A-Za-z0-9_.]*)",
|
|
23
|
+
re.IGNORECASE)
|
|
24
|
+
CALL_RE = re.compile(r"(?<![\w$.:])([A-Za-z_][\w]*)\s*\(")
|
|
25
|
+
|
|
26
|
+
LANG_PATTERNS: Dict[str, Dict[str, Pattern]] = {
|
|
27
|
+
"go": {
|
|
28
|
+
"function": re.compile(
|
|
29
|
+
r"^func\s+(?:\([^)]*\)\s+)?([A-Za-z_]\w*)\s*\(([^)]*)\)",
|
|
30
|
+
re.MULTILINE),
|
|
31
|
+
"class": re.compile(
|
|
32
|
+
r"^type\s+([A-Za-z_]\w*)\s+struct\b", re.MULTILINE),
|
|
33
|
+
"import": re.compile(r'"([\w./-]+)"'),
|
|
34
|
+
"route": re.compile(
|
|
35
|
+
r'\.(?:GET|POST|PUT|DELETE|PATCH|Handle(?:Func)?)\s*\(\s*"([^"]+)"'
|
|
36
|
+
r"\s*,\s*(\w+)?"),
|
|
37
|
+
},
|
|
38
|
+
"java": {
|
|
39
|
+
"function": re.compile(
|
|
40
|
+
r"^[ \t]*(?:public|private|protected)[ \t]+(?:static[ \t]+)?"
|
|
41
|
+
r"(?:final[ \t]+)?[\w<>\[\], ?]+[ \t]+(\w+)\s*\(([^)]*)\)\s*"
|
|
42
|
+
r"(?:throws [\w, ]+)?\s*\{", re.MULTILINE),
|
|
43
|
+
"class": re.compile(
|
|
44
|
+
r"^[ \t]*(?:public\s+)?(?:abstract\s+|final\s+)?"
|
|
45
|
+
r"(?:class|interface|record|enum)\s+(\w+)", re.MULTILINE),
|
|
46
|
+
"import": re.compile(r"^import\s+([\w.]+)", re.MULTILINE),
|
|
47
|
+
"route": re.compile(
|
|
48
|
+
r"@(Get|Post|Put|Delete|Patch|Request)Mapping\s*\(\s*"
|
|
49
|
+
r"(?:value\s*=\s*)?\"([^\"]+)\""),
|
|
50
|
+
},
|
|
51
|
+
"csharp": {
|
|
52
|
+
"function": re.compile(
|
|
53
|
+
r"^[ \t]*(?:public|private|protected|internal)[ \t]+"
|
|
54
|
+
r"(?:static[ \t]+|async[ \t]+|virtual[ \t]+|override[ \t]+)*"
|
|
55
|
+
r"[\w<>\[\], ?]+[ \t]+(\w+)\s*\(([^)]*)\)", re.MULTILINE),
|
|
56
|
+
"class": re.compile(
|
|
57
|
+
r"^[ \t]*(?:public\s+)?(?:abstract\s+|sealed\s+|partial\s+)*"
|
|
58
|
+
r"(?:class|interface|record)\s+(\w+)", re.MULTILINE),
|
|
59
|
+
"import": re.compile(r"^using\s+([\w.]+)", re.MULTILINE),
|
|
60
|
+
"route": re.compile(r"\[Http(Get|Post|Put|Delete|Patch)"
|
|
61
|
+
r"(?:\(\s*\"([^\"]*)\"\s*\))?\]"),
|
|
62
|
+
},
|
|
63
|
+
"php": {
|
|
64
|
+
"function": re.compile(
|
|
65
|
+
r"(?:public\s+|private\s+|protected\s+|static\s+)*"
|
|
66
|
+
r"function\s+&?(\w+)\s*\(([^)]*)\)"),
|
|
67
|
+
"class": re.compile(r"^\s*(?:abstract\s+|final\s+)?class\s+(\w+)",
|
|
68
|
+
re.MULTILINE),
|
|
69
|
+
"import": re.compile(
|
|
70
|
+
r"(?:require|include)(?:_once)?\s*\(?\s*(?:__DIR__\s*\.\s*)?"
|
|
71
|
+
r"['\"]/?([^'\"]+\.php)['\"]"),
|
|
72
|
+
"route": re.compile(
|
|
73
|
+
r"Route::(get|post|put|delete|patch|any)\s*\(\s*['\"]([^'\"]+)['\"]"
|
|
74
|
+
r"(?:\s*,\s*\[?\s*[\w:]*['\"]?\s*,?\s*['\"]?(\w+))?"),
|
|
75
|
+
},
|
|
76
|
+
"ruby": {
|
|
77
|
+
"function": re.compile(r"^\s*def\s+(?:self\.)?(\w+[?!]?)\s*"
|
|
78
|
+
r"(?:\(([^)]*)\))?", re.MULTILINE),
|
|
79
|
+
"class": re.compile(r"^\s*(?:class|module)\s+([A-Z]\w*)",
|
|
80
|
+
re.MULTILINE),
|
|
81
|
+
"import": re.compile(r"^\s*require(?:_relative)?\s+['\"]([^'\"]+)",
|
|
82
|
+
re.MULTILINE),
|
|
83
|
+
"route": re.compile(r"^\s*(get|post|put|delete|patch)\s+['\"]([^'\"]+)"
|
|
84
|
+
r"['\"](?:\s*(?:,\s*to:\s*|=>\s*)['\"]([\w#]+))?",
|
|
85
|
+
re.MULTILINE),
|
|
86
|
+
},
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _line_of(text: str, idx: int) -> int:
|
|
91
|
+
return text.count("\n", 0, idx) + 1
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _brace_end_line(text: str, from_idx: int) -> int:
|
|
95
|
+
"""Line of the `}` closing the first `{` at/after from_idx. 0 if none."""
|
|
96
|
+
i = text.find("{", from_idx)
|
|
97
|
+
if i == -1 or i - from_idx > 300:
|
|
98
|
+
return 0
|
|
99
|
+
depth, n = 0, len(text)
|
|
100
|
+
in_str = None
|
|
101
|
+
while i < n:
|
|
102
|
+
c = text[i]
|
|
103
|
+
if in_str:
|
|
104
|
+
if c == "\\":
|
|
105
|
+
i += 2
|
|
106
|
+
continue
|
|
107
|
+
if c == in_str:
|
|
108
|
+
in_str = None
|
|
109
|
+
elif c in "'\"":
|
|
110
|
+
in_str = c
|
|
111
|
+
elif c == "{":
|
|
112
|
+
depth += 1
|
|
113
|
+
elif c == "}":
|
|
114
|
+
depth -= 1
|
|
115
|
+
if depth == 0:
|
|
116
|
+
return _line_of(text, i)
|
|
117
|
+
i += 1
|
|
118
|
+
return 0
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _ruby_end_line(lines: list, start_line: int) -> int:
|
|
122
|
+
"""Line of the `end` matching a `def` on start_line (1-based). 0 if none."""
|
|
123
|
+
head = lines[start_line - 1]
|
|
124
|
+
indent = len(head) - len(head.lstrip())
|
|
125
|
+
for j in range(start_line, min(len(lines), start_line + 300)):
|
|
126
|
+
line = lines[j]
|
|
127
|
+
if line.strip() == "end" and len(line) - len(line.lstrip()) <= indent:
|
|
128
|
+
return j + 1
|
|
129
|
+
return 0
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def parse_generic(info: FileInfo, text: str, language: str) -> None:
|
|
133
|
+
pats = LANG_PATTERNS[language]
|
|
134
|
+
|
|
135
|
+
for m in pats["import"].finditer(text):
|
|
136
|
+
info.imports.append(m.group(1))
|
|
137
|
+
info.imports = sorted(set(info.imports))[:60]
|
|
138
|
+
|
|
139
|
+
lines = text.split("\n")
|
|
140
|
+
|
|
141
|
+
for m in pats["function"].finditer(text):
|
|
142
|
+
name = m.group(1)
|
|
143
|
+
if name in KEYWORDS or (language == "go" and name == "init"):
|
|
144
|
+
if name in KEYWORDS:
|
|
145
|
+
continue
|
|
146
|
+
params_raw = m.group(2) if m.lastindex and m.lastindex >= 2 else ""
|
|
147
|
+
start_line = _line_of(text, m.start())
|
|
148
|
+
if language == "ruby":
|
|
149
|
+
end_line = _ruby_end_line(lines, start_line)
|
|
150
|
+
else:
|
|
151
|
+
end_line = _brace_end_line(text, m.end())
|
|
152
|
+
if not end_line or end_line < start_line:
|
|
153
|
+
end_line = min(start_line + 60, len(lines)) # fallback guess
|
|
154
|
+
body = "\n".join(lines[start_line - 1:min(end_line, start_line + 120)])
|
|
155
|
+
fn = FunctionInfo(
|
|
156
|
+
name=name,
|
|
157
|
+
params=[p.strip() for p in (params_raw or "").split(",")
|
|
158
|
+
if p.strip()][:10],
|
|
159
|
+
line=start_line, end_line=end_line)
|
|
160
|
+
# route annotation immediately above (Spring / ASP.NET)
|
|
161
|
+
prefix = "\n".join(lines[max(0, start_line - 4):start_line - 1])
|
|
162
|
+
rm = pats["route"].search(prefix)
|
|
163
|
+
if rm:
|
|
164
|
+
g = rm.groups()
|
|
165
|
+
method = (g[0] or "ANY").upper().replace("REQUEST", "ANY")
|
|
166
|
+
path = (g[1] if len(g) > 1 and g[1] else "/")
|
|
167
|
+
fn.routes.append(Route(method=method, path=path, line=start_line))
|
|
168
|
+
for sm in SQL_RE.finditer(body):
|
|
169
|
+
fn.db_refs.append(DbRef(table=sm.group(2), op=sm.group(1).lower(),
|
|
170
|
+
via="sql", line=start_line))
|
|
171
|
+
seen = set()
|
|
172
|
+
for cm in CALL_RE.finditer(body):
|
|
173
|
+
cname = cm.group(1)
|
|
174
|
+
if cname in KEYWORDS or cname == name or cname in seen:
|
|
175
|
+
continue
|
|
176
|
+
seen.add(cname)
|
|
177
|
+
fn.calls.append(cname)
|
|
178
|
+
fn.calls = fn.calls[:40]
|
|
179
|
+
fn.description = describe(fn, body[:400])
|
|
180
|
+
info.functions.append(fn)
|
|
181
|
+
|
|
182
|
+
# file-level routes not attached to a function (Laravel/Sinatra/Gin)
|
|
183
|
+
fn_lines = [(f.line, f.end_line) for f in info.functions]
|
|
184
|
+
for m in pats["route"].finditer(text):
|
|
185
|
+
line = _line_of(text, m.start())
|
|
186
|
+
if any(s <= line <= e for s, e in fn_lines):
|
|
187
|
+
continue
|
|
188
|
+
g = m.groups()
|
|
189
|
+
method = (g[0] or "ANY").upper().replace("REQUEST", "ANY")
|
|
190
|
+
path = g[1] if len(g) > 1 and g[1] else "/"
|
|
191
|
+
handler = g[2] if len(g) > 2 and g[2] else ""
|
|
192
|
+
if "#" in handler: # rails 'users#index'
|
|
193
|
+
handler = handler.split("#")[-1]
|
|
194
|
+
info.routes.append(Route(method=method, path=path, line=line,
|
|
195
|
+
handler=handler))
|
|
196
|
+
|
|
197
|
+
if language in ("php", "ruby"):
|
|
198
|
+
extract_toplevel(info, text)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
PAGE_LINK_RE = re.compile(
|
|
202
|
+
r"(?:redirect|url|location|navigate)\s*\(\s*['\"]([\w\-./]+\.(?:php|html?))"
|
|
203
|
+
r"|href\s*=\s*['\"]([\w\-./]+\.(?:php|html?))"
|
|
204
|
+
r"|action\s*=\s*['\"]([\w\-./]+\.php)"
|
|
205
|
+
r"|Location:\s*([\w\-./]+\.php)", re.IGNORECASE)
|
|
206
|
+
|
|
207
|
+
TOPLEVEL_NOISE = KEYWORDS | {
|
|
208
|
+
"echo", "isset", "empty", "unset", "die", "exit", "list", "array",
|
|
209
|
+
"define", "defined", "declare", "compact", "extract", "eval", "clone",
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def extract_toplevel(info: FileInfo, text: str) -> None:
|
|
214
|
+
"""Calls made outside any extracted function (classic script-style code,
|
|
215
|
+
e.g. a PHP page calling auth_boot()), plus page-navigation links."""
|
|
216
|
+
fn_ranges = [(f.line, max(f.end_line, f.line)) for f in info.functions]
|
|
217
|
+
|
|
218
|
+
def outside(line: int) -> bool:
|
|
219
|
+
return not any(s <= line <= e for s, e in fn_ranges)
|
|
220
|
+
|
|
221
|
+
seen = set()
|
|
222
|
+
for m in CALL_RE.finditer(text):
|
|
223
|
+
name = m.group(1)
|
|
224
|
+
if name in TOPLEVEL_NOISE or name in seen:
|
|
225
|
+
continue
|
|
226
|
+
if outside(_line_of(text, m.start())):
|
|
227
|
+
seen.add(name)
|
|
228
|
+
info.calls.append(name)
|
|
229
|
+
info.calls = info.calls[:40]
|
|
230
|
+
|
|
231
|
+
seen_l = set()
|
|
232
|
+
for m in PAGE_LINK_RE.finditer(text):
|
|
233
|
+
target = next(g for g in m.groups() if g)
|
|
234
|
+
if target not in seen_l:
|
|
235
|
+
seen_l.add(target)
|
|
236
|
+
info.links.append(target)
|
|
237
|
+
info.links = info.links[:30]
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
CREATE_TABLE_RE = re.compile(
|
|
241
|
+
r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[`\"\[]?"
|
|
242
|
+
r"([A-Za-z_][\w.]*)[`\"\]]?\s*\(([\s\S]*?)\)\s*;",
|
|
243
|
+
re.IGNORECASE)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def parse_sql(info: FileInfo, text: str) -> None:
|
|
247
|
+
for m in CREATE_TABLE_RE.finditer(text):
|
|
248
|
+
cols = []
|
|
249
|
+
for raw in m.group(2).split(","):
|
|
250
|
+
token = raw.strip().split()
|
|
251
|
+
if token and not token[0].upper() in (
|
|
252
|
+
"PRIMARY", "FOREIGN", "UNIQUE", "CONSTRAINT", "KEY",
|
|
253
|
+
"INDEX", "CHECK"):
|
|
254
|
+
cols.append(token[0].strip('`"[]'))
|
|
255
|
+
info.tables.append(TableInfo(name=m.group(1),
|
|
256
|
+
fields=cols[:30],
|
|
257
|
+
line=_line_of(text, m.start()),
|
|
258
|
+
source=info.path))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
SECRET_KEY_RE = re.compile(
|
|
262
|
+
r"(PASS(WORD)?|SECRET|TOKEN|API[_-]?KEY|PRIVATE|CREDENTIAL|AUTH)",
|
|
263
|
+
re.IGNORECASE)
|
|
264
|
+
DB_KEY_RE = re.compile(
|
|
265
|
+
r"(DB|DATABASE|POSTGRES|MYSQL|MONGO|REDIS|SQL|CONN)", re.IGNORECASE)
|
|
266
|
+
URL_CREDS_RE = re.compile(r"://([^:/@\s]+):([^@\s]+)@")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def parse_config(info: FileInfo, text: str) -> None:
|
|
270
|
+
"""Look for DB connection settings in config files. Mask every secret."""
|
|
271
|
+
for i, line in enumerate(text.splitlines()[:400], 1):
|
|
272
|
+
stripped = line.strip()
|
|
273
|
+
if not stripped or stripped.startswith(("#", ";", "//")):
|
|
274
|
+
continue
|
|
275
|
+
m = re.match(r"^\s*[\"']?([\w.\-]+)[\"']?\s*[:=]\s*(.+)$", stripped)
|
|
276
|
+
if not m:
|
|
277
|
+
continue
|
|
278
|
+
key, value = m.group(1), m.group(2).strip().strip("\"',")
|
|
279
|
+
if not (DB_KEY_RE.search(key) or SECRET_KEY_RE.search(key)):
|
|
280
|
+
continue
|
|
281
|
+
if SECRET_KEY_RE.search(key):
|
|
282
|
+
value = "•••masked•••"
|
|
283
|
+
else:
|
|
284
|
+
value = URL_CREDS_RE.sub("://***:***@", value)
|
|
285
|
+
if len(value) > 60:
|
|
286
|
+
value = value[:57] + "..."
|
|
287
|
+
info.db_config.append(f"{key} = {value}")
|
|
288
|
+
info.db_config = info.db_config[:20]
|