universal-dev-security 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- universal_dev_security/__init__.py +3 -0
- universal_dev_security/__main__.py +4 -0
- universal_dev_security/cli.py +376 -0
- universal_dev_security/cli_extra.py +244 -0
- universal_dev_security/config.py +171 -0
- universal_dev_security/detector.py +201 -0
- universal_dev_security/engine.py +203 -0
- universal_dev_security/findings.py +199 -0
- universal_dev_security/fixes.py +138 -0
- universal_dev_security/gitguard.py +269 -0
- universal_dev_security/history.py +149 -0
- universal_dev_security/initializer.py +112 -0
- universal_dev_security/integrations/__init__.py +46 -0
- universal_dev_security/integrations/agents.py +45 -0
- universal_dev_security/integrations/claude_code.py +192 -0
- universal_dev_security/integrations/github.py +149 -0
- universal_dev_security/integrations/vscode.py +44 -0
- universal_dev_security/policy.py +384 -0
- universal_dev_security/projectfiles.py +80 -0
- universal_dev_security/recovery.py +168 -0
- universal_dev_security/redaction.py +132 -0
- universal_dev_security/reports.py +466 -0
- universal_dev_security/scanner.py +36 -0
- universal_dev_security/scanners/__init__.py +36 -0
- universal_dev_security/scanners/base.py +269 -0
- universal_dev_security/scanners/gitleaks.py +132 -0
- universal_dev_security/scanners/osv.py +156 -0
- universal_dev_security/scanners/semgrep.py +130 -0
- universal_dev_security/scanners/trivy.py +139 -0
- universal_dev_security/testing.py +100 -0
- universal_dev_security/updates.py +84 -0
- universal_dev_security-1.0.0.dist-info/METADATA +128 -0
- universal_dev_security-1.0.0.dist-info/RECORD +37 -0
- universal_dev_security-1.0.0.dist-info/WHEEL +5 -0
- universal_dev_security-1.0.0.dist-info/entry_points.txt +2 -0
- universal_dev_security-1.0.0.dist-info/licenses/LICENSE +163 -0
- universal_dev_security-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Load and validate project configuration (``.uds/config.toml``).
|
|
2
|
+
|
|
3
|
+
Project configuration can only *add* restrictions. Settings that would weaken
|
|
4
|
+
the mandatory global policy are ignored and reported as warnings.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import tomllib
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
CONFIG_RELATIVE_PATH = Path(".uds") / "config.toml"
|
|
14
|
+
|
|
15
|
+
KNOWN_SCANNERS = ("gitleaks", "semgrep", "osv-scanner", "trivy")
|
|
16
|
+
FAIL_ON_LEVELS = ("none", "unclassified", "low", "medium", "high", "critical")
|
|
17
|
+
SECURITY_MODES = ("balanced", "strict")
|
|
18
|
+
|
|
19
|
+
# Mandatory settings: the only accepted value is the safe one.
|
|
20
|
+
MANDATORY_TRUE = (
|
|
21
|
+
"require_user_approval_for_high_risk_changes",
|
|
22
|
+
"require_user_approval_for_git_push",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ConfigError(ValueError):
|
|
27
|
+
"""Raised when the configuration file cannot be parsed."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class ProjectConfig:
|
|
32
|
+
path: Path | None = None
|
|
33
|
+
exists: bool = False
|
|
34
|
+
project_name: str = ""
|
|
35
|
+
profile: str = "auto"
|
|
36
|
+
security_mode: str = "balanced"
|
|
37
|
+
allow_automatic_low_risk_fixes: bool = True
|
|
38
|
+
require_user_approval_for_high_risk_changes: bool = True
|
|
39
|
+
require_user_approval_for_git_push: bool = True
|
|
40
|
+
fail_on: str = "none"
|
|
41
|
+
deny_command_patterns: list[str] = field(default_factory=list)
|
|
42
|
+
approval_command_patterns: list[str] = field(default_factory=list)
|
|
43
|
+
protected_paths: list[str] = field(default_factory=list)
|
|
44
|
+
enabled_scanners: list[str] = field(default_factory=lambda: list(KNOWN_SCANNERS))
|
|
45
|
+
allow_network: bool = False
|
|
46
|
+
semgrep_configs: list[str] = field(default_factory=list)
|
|
47
|
+
scanner_timeout_seconds: int = 600
|
|
48
|
+
exclude_paths: list[str] = field(default_factory=list)
|
|
49
|
+
test_command: list[str] = field(default_factory=list)
|
|
50
|
+
test_timeout_seconds: int = 900
|
|
51
|
+
history_limit: int = 50
|
|
52
|
+
warnings: list[str] = field(default_factory=list)
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> dict[str, Any]:
|
|
55
|
+
return {
|
|
56
|
+
"path": str(self.path) if self.path else None,
|
|
57
|
+
"exists": self.exists,
|
|
58
|
+
"project": {"name": self.project_name, "profile": self.profile},
|
|
59
|
+
"policy": {
|
|
60
|
+
"security_mode": self.security_mode,
|
|
61
|
+
"allow_automatic_low_risk_fixes": self.allow_automatic_low_risk_fixes,
|
|
62
|
+
"require_user_approval_for_high_risk_changes": self.require_user_approval_for_high_risk_changes,
|
|
63
|
+
"require_user_approval_for_git_push": self.require_user_approval_for_git_push,
|
|
64
|
+
"fail_on": self.fail_on,
|
|
65
|
+
"deny_command_patterns": self.deny_command_patterns,
|
|
66
|
+
"approval_command_patterns": self.approval_command_patterns,
|
|
67
|
+
"protected_paths": self.protected_paths,
|
|
68
|
+
},
|
|
69
|
+
"scanners": {
|
|
70
|
+
"enabled": self.enabled_scanners,
|
|
71
|
+
"allow_network": self.allow_network,
|
|
72
|
+
"semgrep_configs": self.semgrep_configs,
|
|
73
|
+
"timeout_seconds": self.scanner_timeout_seconds,
|
|
74
|
+
"exclude_paths": self.exclude_paths,
|
|
75
|
+
},
|
|
76
|
+
"testing": {"command": self.test_command, "timeout_seconds": self.test_timeout_seconds},
|
|
77
|
+
"history": {"limit": self.history_limit},
|
|
78
|
+
"warnings": self.warnings,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _str_list(value: Any, key: str, warnings: list[str], max_items: int = 200) -> list[str]:
|
|
83
|
+
if value is None:
|
|
84
|
+
return []
|
|
85
|
+
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
86
|
+
warnings.append(f"{key} must be a list of strings; ignored.")
|
|
87
|
+
return []
|
|
88
|
+
return [item for item in value if item.strip()][:max_items]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _bounded_int(value: Any, key: str, default: int, low: int, high: int, warnings: list[str]) -> int:
|
|
92
|
+
if value is None:
|
|
93
|
+
return default
|
|
94
|
+
if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high:
|
|
95
|
+
warnings.append(f"{key} must be an integer between {low} and {high}; using {default}.")
|
|
96
|
+
return default
|
|
97
|
+
return value
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def load_config(project_root: Path) -> ProjectConfig:
|
|
101
|
+
root = project_root.expanduser().resolve()
|
|
102
|
+
path = root / CONFIG_RELATIVE_PATH
|
|
103
|
+
config = ProjectConfig(path=path)
|
|
104
|
+
if not path.is_file():
|
|
105
|
+
return config
|
|
106
|
+
try:
|
|
107
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
108
|
+
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
|
|
109
|
+
raise ConfigError(f"Could not parse {path}: {type(exc).__name__}") from exc
|
|
110
|
+
config.exists = True
|
|
111
|
+
warnings = config.warnings
|
|
112
|
+
|
|
113
|
+
project = data.get("project", {}) if isinstance(data.get("project"), dict) else {}
|
|
114
|
+
config.project_name = str(project.get("name", ""))[:200]
|
|
115
|
+
config.profile = str(project.get("profile", "auto"))[:50]
|
|
116
|
+
|
|
117
|
+
policy = data.get("policy", {}) if isinstance(data.get("policy"), dict) else {}
|
|
118
|
+
mode = policy.get("security_mode", "balanced")
|
|
119
|
+
if mode in SECURITY_MODES:
|
|
120
|
+
config.security_mode = mode
|
|
121
|
+
else:
|
|
122
|
+
warnings.append(f"Unknown security_mode {mode!r}; using 'balanced'.")
|
|
123
|
+
if isinstance(policy.get("allow_automatic_low_risk_fixes"), bool):
|
|
124
|
+
config.allow_automatic_low_risk_fixes = policy["allow_automatic_low_risk_fixes"]
|
|
125
|
+
for key in MANDATORY_TRUE:
|
|
126
|
+
if key in policy and policy[key] is not True:
|
|
127
|
+
warnings.append(
|
|
128
|
+
f"policy.{key} cannot be disabled: the mandatory global policy requires it. Setting ignored."
|
|
129
|
+
)
|
|
130
|
+
fail_on = policy.get("fail_on", "none")
|
|
131
|
+
if fail_on in FAIL_ON_LEVELS:
|
|
132
|
+
config.fail_on = fail_on
|
|
133
|
+
else:
|
|
134
|
+
warnings.append(f"Unknown policy.fail_on {fail_on!r}; using 'none'.")
|
|
135
|
+
config.deny_command_patterns = _str_list(policy.get("deny_command_patterns"), "policy.deny_command_patterns", warnings)
|
|
136
|
+
config.approval_command_patterns = _str_list(
|
|
137
|
+
policy.get("approval_command_patterns"), "policy.approval_command_patterns", warnings
|
|
138
|
+
)
|
|
139
|
+
config.protected_paths = _str_list(policy.get("protected_paths"), "policy.protected_paths", warnings)
|
|
140
|
+
for key in ("allow_commands", "allowed_commands", "disable_rules", "disabled_rules"):
|
|
141
|
+
if key in policy:
|
|
142
|
+
warnings.append(f"policy.{key} is not supported: global rules cannot be disabled by project configuration.")
|
|
143
|
+
|
|
144
|
+
scanners = data.get("scanners", {}) if isinstance(data.get("scanners"), dict) else {}
|
|
145
|
+
if "enabled" in scanners:
|
|
146
|
+
enabled = _str_list(scanners.get("enabled"), "scanners.enabled", warnings)
|
|
147
|
+
unknown = [name for name in enabled if name not in KNOWN_SCANNERS]
|
|
148
|
+
if unknown:
|
|
149
|
+
warnings.append(f"Unknown scanners ignored: {', '.join(unknown)}")
|
|
150
|
+
config.enabled_scanners = [name for name in enabled if name in KNOWN_SCANNERS]
|
|
151
|
+
if "gitleaks" not in config.enabled_scanners:
|
|
152
|
+
# Secret scanning is a mandatory global control.
|
|
153
|
+
config.enabled_scanners.insert(0, "gitleaks")
|
|
154
|
+
warnings.append("gitleaks secret scanning is mandatory and was re-enabled.")
|
|
155
|
+
if isinstance(scanners.get("allow_network"), bool):
|
|
156
|
+
config.allow_network = scanners["allow_network"]
|
|
157
|
+
config.semgrep_configs = _str_list(scanners.get("semgrep_configs"), "scanners.semgrep_configs", warnings, 20)
|
|
158
|
+
config.scanner_timeout_seconds = _bounded_int(
|
|
159
|
+
scanners.get("timeout_seconds"), "scanners.timeout_seconds", 600, 10, 7200, warnings
|
|
160
|
+
)
|
|
161
|
+
config.exclude_paths = _str_list(scanners.get("exclude_paths"), "scanners.exclude_paths", warnings)
|
|
162
|
+
|
|
163
|
+
testing = data.get("testing", {}) if isinstance(data.get("testing"), dict) else {}
|
|
164
|
+
config.test_command = _str_list(testing.get("command"), "testing.command", warnings, 50)
|
|
165
|
+
config.test_timeout_seconds = _bounded_int(
|
|
166
|
+
testing.get("timeout_seconds"), "testing.timeout_seconds", 900, 10, 7200, warnings
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
history = data.get("history", {}) if isinstance(data.get("history"), dict) else {}
|
|
170
|
+
config.history_limit = _bounded_int(history.get("limit"), "history.limit", 50, 1, 1000, warnings)
|
|
171
|
+
return config
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Conservative, local-only project technology detection and scanning profiles."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from universal_dev_security.projectfiles import ProjectInventory, build_inventory
|
|
9
|
+
|
|
10
|
+
# Lockfiles and manifests that dependency scanners (OSV-Scanner, Trivy) understand.
|
|
11
|
+
DEPENDENCY_FILES = {
|
|
12
|
+
"requirements.txt", "poetry.lock", "pipfile.lock", "pdm.lock", "uv.lock", "pyproject.toml", "setup.py",
|
|
13
|
+
"package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "package.json",
|
|
14
|
+
"composer.lock", "composer.json", "gemfile.lock", "cargo.lock", "go.mod", "go.sum",
|
|
15
|
+
"pom.xml", "gradle.lockfile", "build.gradle", "build.gradle.kts", "packages.lock.json",
|
|
16
|
+
"packages.config", "pubspec.lock", "mix.lock", "conan.lock",
|
|
17
|
+
}
|
|
18
|
+
DEPENDENCY_SUFFIXES = {".csproj", ".fsproj", ".vbproj"}
|
|
19
|
+
|
|
20
|
+
CODE_SUFFIXES = {
|
|
21
|
+
".py", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".php", ".java", ".kt", ".cs", ".go", ".rb",
|
|
22
|
+
".rs", ".scala", ".swift", ".c", ".cc", ".cpp", ".h", ".hpp", ".sh", ".ps1", ".vue", ".svelte",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
IAC_NAMES = {"dockerfile", "containerfile", "docker-compose.yml", "docker-compose.yaml", "compose.yml",
|
|
26
|
+
"compose.yaml", "chart.yaml", "kustomization.yaml"}
|
|
27
|
+
IAC_SUFFIXES = {".tf", ".tfvars", ".bicep", ".hcl"}
|
|
28
|
+
|
|
29
|
+
# Technology profile -> scanners that are meaningful + optional Semgrep rule packs.
|
|
30
|
+
# Rule pack names refer to the public Semgrep Registry (https://semgrep.dev/explore).
|
|
31
|
+
PROFILES: dict[str, dict[str, Any]] = {
|
|
32
|
+
"generic": {"scanners": ["gitleaks"], "semgrep": []},
|
|
33
|
+
"python": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/python"]},
|
|
34
|
+
"javascript": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/javascript"]},
|
|
35
|
+
"typescript": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/typescript"]},
|
|
36
|
+
"node": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/nodejs"]},
|
|
37
|
+
"react": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/react"]},
|
|
38
|
+
"nextjs": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/react"]},
|
|
39
|
+
"php": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/php"]},
|
|
40
|
+
"wordpress": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/php"]},
|
|
41
|
+
"woocommerce": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/php"]},
|
|
42
|
+
"java": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/java"]},
|
|
43
|
+
"dotnet": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/csharp"]},
|
|
44
|
+
"go": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/golang"]},
|
|
45
|
+
"rust": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": []},
|
|
46
|
+
"ruby": {"scanners": ["gitleaks", "semgrep", "osv-scanner", "trivy"], "semgrep": ["p/ruby"]},
|
|
47
|
+
"docker": {"scanners": ["gitleaks", "semgrep", "trivy"], "semgrep": ["p/dockerfile"]},
|
|
48
|
+
"terraform": {"scanners": ["gitleaks", "semgrep", "trivy"], "semgrep": ["p/terraform"]},
|
|
49
|
+
"kubernetes": {"scanners": ["gitleaks", "semgrep", "trivy"], "semgrep": ["p/kubernetes"]},
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _read_json(path: Path, limit: int = 2_000_000) -> dict[str, Any]:
|
|
54
|
+
try:
|
|
55
|
+
if path.stat().st_size > limit:
|
|
56
|
+
return {}
|
|
57
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
58
|
+
return data if isinstance(data, dict) else {}
|
|
59
|
+
except (OSError, ValueError, UnicodeDecodeError):
|
|
60
|
+
return {}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _package_json_dependencies(root: Path, inventory: ProjectInventory) -> set[str]:
|
|
64
|
+
deps: set[str] = set()
|
|
65
|
+
for relative in inventory.find("package.json")[:25]:
|
|
66
|
+
data = _read_json(root / relative)
|
|
67
|
+
for key in ("dependencies", "devDependencies", "peerDependencies"):
|
|
68
|
+
section = data.get(key)
|
|
69
|
+
if isinstance(section, dict):
|
|
70
|
+
deps.update(str(name).lower() for name in section)
|
|
71
|
+
return deps
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _composer_requirements(root: Path, inventory: ProjectInventory) -> set[str]:
|
|
75
|
+
deps: set[str] = set()
|
|
76
|
+
for relative in inventory.find("composer.json")[:25]:
|
|
77
|
+
data = _read_json(root / relative)
|
|
78
|
+
for key in ("require", "require-dev"):
|
|
79
|
+
section = data.get(key)
|
|
80
|
+
if isinstance(section, dict):
|
|
81
|
+
deps.update(str(name).lower() for name in section)
|
|
82
|
+
return deps
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def detect_technologies(root: Path, inventory: ProjectInventory) -> dict[str, list[str]]:
|
|
86
|
+
"""Return {technology: [evidence, ...]} from file markers and manifests."""
|
|
87
|
+
found: dict[str, list[str]] = {}
|
|
88
|
+
|
|
89
|
+
def add(tech: str, evidence: str) -> None:
|
|
90
|
+
found.setdefault(tech, [])
|
|
91
|
+
if evidence not in found[tech] and len(found[tech]) < 5:
|
|
92
|
+
found[tech].append(evidence)
|
|
93
|
+
|
|
94
|
+
for name in ("pyproject.toml", "requirements.txt", "Pipfile", "setup.py", "setup.cfg", "poetry.lock", "uv.lock"):
|
|
95
|
+
for path in inventory.find(name)[:2]:
|
|
96
|
+
add("python", path)
|
|
97
|
+
if inventory.has_suffix(".py") and "python" not in found:
|
|
98
|
+
add("python", "*.py source files")
|
|
99
|
+
|
|
100
|
+
js_deps = _package_json_dependencies(root, inventory)
|
|
101
|
+
for path in inventory.find("package.json")[:2]:
|
|
102
|
+
add("node", path)
|
|
103
|
+
add("javascript", path)
|
|
104
|
+
if inventory.has_suffix(".js", ".jsx", ".mjs", ".cjs"):
|
|
105
|
+
add("javascript", "*.js source files")
|
|
106
|
+
if inventory.find("tsconfig.json") or inventory.has_suffix(".ts", ".tsx") or "typescript" in js_deps:
|
|
107
|
+
add("typescript", (inventory.find("tsconfig.json") or ["*.ts source files"])[0])
|
|
108
|
+
if "react" in js_deps or "react-dom" in js_deps:
|
|
109
|
+
add("react", "package.json dependency: react")
|
|
110
|
+
if "next" in js_deps or any(inventory.find(n) for n in ("next.config.js", "next.config.mjs", "next.config.ts")):
|
|
111
|
+
add("nextjs", "package.json dependency: next" if "next" in js_deps else "next.config.*")
|
|
112
|
+
add("react", "Next.js implies React")
|
|
113
|
+
|
|
114
|
+
composer = _composer_requirements(root, inventory)
|
|
115
|
+
for path in inventory.find("composer.json")[:2]:
|
|
116
|
+
add("php", path)
|
|
117
|
+
if inventory.has_suffix(".php"):
|
|
118
|
+
add("php", "*.php source files")
|
|
119
|
+
wp_markers = inventory.find("wp-config.php") + inventory.find("wp-config-sample.php") + inventory.find("wp-load.php")
|
|
120
|
+
if wp_markers or inventory.has_path_part("wp-content") or any("wordpress" in d for d in composer):
|
|
121
|
+
add("wordpress", (wp_markers or ["wp-content/"])[0])
|
|
122
|
+
add("php", "WordPress implies PHP")
|
|
123
|
+
plugin_headers = [p for p in inventory.files if p.lower().endswith("woocommerce.php")]
|
|
124
|
+
if (inventory.has_path_part("wp-content/plugins/woocommerce") or plugin_headers
|
|
125
|
+
or any("woocommerce" in d for d in composer)):
|
|
126
|
+
add("woocommerce", (plugin_headers or ["wp-content/plugins/woocommerce/"])[0])
|
|
127
|
+
|
|
128
|
+
for name in ("pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle"):
|
|
129
|
+
for path in inventory.find(name)[:2]:
|
|
130
|
+
add("java", path)
|
|
131
|
+
if inventory.has_suffix(".java"):
|
|
132
|
+
add("java", "*.java source files")
|
|
133
|
+
|
|
134
|
+
for path in [p for p in inventory.files if p.lower().endswith((".sln", ".csproj", ".fsproj", ".vbproj"))][:3]:
|
|
135
|
+
add("dotnet", path)
|
|
136
|
+
|
|
137
|
+
for path in inventory.find("go.mod")[:2]:
|
|
138
|
+
add("go", path)
|
|
139
|
+
for path in inventory.find("Cargo.toml")[:2]:
|
|
140
|
+
add("rust", path)
|
|
141
|
+
for path in inventory.find("Gemfile")[:2]:
|
|
142
|
+
add("ruby", path)
|
|
143
|
+
|
|
144
|
+
docker = [p for p in inventory.files if p.rsplit("/", 1)[-1].lower() in IAC_NAMES - {"chart.yaml", "kustomization.yaml"}
|
|
145
|
+
or p.rsplit("/", 1)[-1].lower().startswith("dockerfile")]
|
|
146
|
+
for path in docker[:3]:
|
|
147
|
+
add("docker", path)
|
|
148
|
+
for path in [p for p in inventory.files if p.lower().endswith((".tf", ".tfvars"))][:3]:
|
|
149
|
+
add("terraform", path)
|
|
150
|
+
for path in (inventory.find("Chart.yaml") + inventory.find("kustomization.yaml"))[:3]:
|
|
151
|
+
add("kubernetes", path)
|
|
152
|
+
return found
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def has_dependency_manifests(inventory: ProjectInventory) -> bool:
|
|
156
|
+
return any(name.lower() in DEPENDENCY_FILES for name in inventory.names) or inventory.has_suffix(*DEPENDENCY_SUFFIXES)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def has_source_code(inventory: ProjectInventory) -> bool:
|
|
160
|
+
return inventory.has_suffix(*CODE_SUFFIXES)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def has_iac(inventory: ProjectInventory) -> bool:
|
|
164
|
+
names = {n.lower() for n in inventory.names}
|
|
165
|
+
return bool(names & IAC_NAMES) or any(n.startswith("dockerfile") for n in names) or inventory.has_suffix(*IAC_SUFFIXES)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def recommended_semgrep_configs(technologies: list[str]) -> list[str]:
|
|
169
|
+
configs = ["p/default"]
|
|
170
|
+
for tech in technologies:
|
|
171
|
+
for config in PROFILES.get(tech, {}).get("semgrep", []):
|
|
172
|
+
if config not in configs:
|
|
173
|
+
configs.append(config)
|
|
174
|
+
return configs
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def detect_project(path: Path, inventory: ProjectInventory | None = None) -> dict[str, Any]:
|
|
178
|
+
target = path.expanduser().resolve()
|
|
179
|
+
if not target.exists() or not target.is_dir():
|
|
180
|
+
raise ValueError(f"Project path is not an existing directory: {target}")
|
|
181
|
+
inventory = inventory or build_inventory(target)
|
|
182
|
+
technologies = detect_technologies(target, inventory)
|
|
183
|
+
detected = sorted(technologies) or ["generic"]
|
|
184
|
+
scanners: list[str] = []
|
|
185
|
+
for tech in detected:
|
|
186
|
+
for scanner in PROFILES.get(tech, PROFILES["generic"])["scanners"]:
|
|
187
|
+
if scanner not in scanners:
|
|
188
|
+
scanners.append(scanner)
|
|
189
|
+
return {
|
|
190
|
+
"path": str(target),
|
|
191
|
+
"detected": detected,
|
|
192
|
+
"evidence": technologies,
|
|
193
|
+
"recommended_scanners": scanners,
|
|
194
|
+
"semgrep_configs": recommended_semgrep_configs(detected) if "semgrep" in scanners else [],
|
|
195
|
+
"dependency_manifests": has_dependency_manifests(inventory),
|
|
196
|
+
"infrastructure_as_code": has_iac(inventory),
|
|
197
|
+
"files_inventoried": len(inventory.files),
|
|
198
|
+
"inventory_truncated": inventory.truncated,
|
|
199
|
+
"detector": "marker-based",
|
|
200
|
+
"network_used": False,
|
|
201
|
+
}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Scan orchestration: run applicable scanners and aggregate honest results."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import uuid
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Callable
|
|
8
|
+
|
|
9
|
+
from universal_dev_security import __version__
|
|
10
|
+
from universal_dev_security.config import FAIL_ON_LEVELS, ProjectConfig
|
|
11
|
+
from universal_dev_security.detector import detect_project
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import shutil
|
|
15
|
+
import subprocess
|
|
16
|
+
|
|
17
|
+
from universal_dev_security.findings import Category, Finding, ScanResult, ScanStatus, Severity, severity_counts
|
|
18
|
+
from universal_dev_security.projectfiles import build_inventory
|
|
19
|
+
from universal_dev_security.scanners import get_scanner
|
|
20
|
+
from universal_dev_security.scanners.base import ScannerExecutionError, ScanOptions, run_tool
|
|
21
|
+
|
|
22
|
+
SCHEMA_VERSION = "2.0"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def overall_status(results: list[ScanResult]) -> str:
|
|
26
|
+
"""Aggregate scanner states without overstating coverage.
|
|
27
|
+
|
|
28
|
+
* ``not_scanned`` - no scanner executed.
|
|
29
|
+
* ``findings`` - at least one executed scanner returned findings.
|
|
30
|
+
* ``error`` - no findings, but at least one scanner failed.
|
|
31
|
+
* ``incomplete`` - no findings or errors, but an applicable scanner did not run.
|
|
32
|
+
* ``clean`` - every applicable scanner executed and returned nothing.
|
|
33
|
+
"""
|
|
34
|
+
executed = [r for r in results if r.status.executed]
|
|
35
|
+
if not executed:
|
|
36
|
+
return "not_scanned"
|
|
37
|
+
if any(r.status is ScanStatus.FINDINGS for r in results):
|
|
38
|
+
return "findings"
|
|
39
|
+
if any(r.status is ScanStatus.ERROR for r in results):
|
|
40
|
+
return "error"
|
|
41
|
+
if any(r.status in (ScanStatus.NOT_INSTALLED, ScanStatus.SKIPPED) for r in results):
|
|
42
|
+
return "incomplete"
|
|
43
|
+
return "clean"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def scope_fingerprint(extra_excludes: list[str]) -> str:
|
|
47
|
+
"""Identify the scan scope (built-in and configured exclusions).
|
|
48
|
+
|
|
49
|
+
When it differs between two scans, findings that disappeared may have been
|
|
50
|
+
excluded rather than fixed, and comparisons must say so.
|
|
51
|
+
"""
|
|
52
|
+
from universal_dev_security.projectfiles import DEFAULT_EXCLUDED_DIRS
|
|
53
|
+
from universal_dev_security.scanners.gitleaks import SECRET_SCAN_EXCLUDES
|
|
54
|
+
|
|
55
|
+
basis = json.dumps([sorted(DEFAULT_EXCLUDED_DIRS), list(SECRET_SCAN_EXCLUDES), sorted(extra_excludes)])
|
|
56
|
+
return hashlib.sha256(basis.encode("utf-8")).hexdigest()[:16]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def git_tracked_files(root: Path) -> set[str] | None:
|
|
60
|
+
"""Project-relative paths tracked by Git, or None when ``root`` is not inside a Git work tree."""
|
|
61
|
+
git = shutil.which("git")
|
|
62
|
+
if not git:
|
|
63
|
+
return None
|
|
64
|
+
try:
|
|
65
|
+
completed = run_tool([git, "ls-files", "-z"], cwd=root, timeout=120)
|
|
66
|
+
except (OSError, subprocess.SubprocessError, ScannerExecutionError):
|
|
67
|
+
return None
|
|
68
|
+
if completed.returncode != 0:
|
|
69
|
+
return None
|
|
70
|
+
return {path.replace("\\", "/") for path in completed.stdout.split("\0") if path}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def annotate_git_exposure(root: Path, findings: list[Finding]) -> None:
|
|
74
|
+
"""Mark secret findings as committed (tracked by Git) or local-only.
|
|
75
|
+
|
|
76
|
+
A secret in a tracked file has been (or will be) shared through the
|
|
77
|
+
repository and must be rotated; one in an untracked or ignored local file
|
|
78
|
+
has not left the machine through Git.
|
|
79
|
+
"""
|
|
80
|
+
secrets = [f for f in findings if f.category is Category.SECRET]
|
|
81
|
+
if not secrets:
|
|
82
|
+
return
|
|
83
|
+
tracked = git_tracked_files(root)
|
|
84
|
+
for finding in secrets:
|
|
85
|
+
if finding.commit:
|
|
86
|
+
finding.tracked_in_git = True # found in Git history
|
|
87
|
+
elif tracked is not None:
|
|
88
|
+
finding.tracked_in_git = finding.file in tracked
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def finding_sort_key(finding: Finding) -> tuple[int, int, str, str, int]:
|
|
92
|
+
"""Committed secrets first, then by severity (critical → unclassified), tool, file, line."""
|
|
93
|
+
exposed = 0 if (finding.category is Category.SECRET and finding.tracked_in_git) else 1
|
|
94
|
+
return (exposed, -finding.severity.rank, finding.tool, finding.file, finding.start_line or 0)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def threshold_reached(findings: list[Finding], fail_on: str) -> bool:
|
|
98
|
+
if fail_on not in FAIL_ON_LEVELS or fail_on == "none":
|
|
99
|
+
return False
|
|
100
|
+
if fail_on == "unclassified":
|
|
101
|
+
return bool(findings)
|
|
102
|
+
threshold = Severity(fail_on).rank
|
|
103
|
+
# Unclassified secrets are always treated as blocking once a threshold is set:
|
|
104
|
+
# an exposed credential is never "low risk" just because it lacks a label.
|
|
105
|
+
return any(f.severity.rank >= threshold or (f.category.value == "secret") for f in findings)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def run_scan(
|
|
109
|
+
target: Path,
|
|
110
|
+
config: ProjectConfig | None = None,
|
|
111
|
+
*,
|
|
112
|
+
tools: list[str] | None = None,
|
|
113
|
+
allow_network: bool | None = None,
|
|
114
|
+
progress: Callable[[str], None] | None = None,
|
|
115
|
+
) -> dict[str, Any]:
|
|
116
|
+
root = target.expanduser().resolve()
|
|
117
|
+
if not root.is_dir():
|
|
118
|
+
raise ValueError(f"Scan target is not an existing directory: {root}")
|
|
119
|
+
config = config or ProjectConfig()
|
|
120
|
+
started = datetime.now(timezone.utc)
|
|
121
|
+
inventory = build_inventory(root, extra_excludes=config.exclude_paths)
|
|
122
|
+
project = detect_project(root, inventory)
|
|
123
|
+
options = ScanOptions(
|
|
124
|
+
timeout_seconds=config.scanner_timeout_seconds,
|
|
125
|
+
allow_network=config.allow_network if allow_network is None else allow_network,
|
|
126
|
+
semgrep_configs=config.semgrep_configs,
|
|
127
|
+
exclude_paths=config.exclude_paths,
|
|
128
|
+
technologies=project["detected"],
|
|
129
|
+
)
|
|
130
|
+
requested = tools or config.enabled_scanners
|
|
131
|
+
results: list[ScanResult] = []
|
|
132
|
+
for name in requested:
|
|
133
|
+
adapter = get_scanner(name)
|
|
134
|
+
if progress:
|
|
135
|
+
progress(f"Running {adapter.display_name}...")
|
|
136
|
+
results.append(adapter.scan(root, inventory, options))
|
|
137
|
+
|
|
138
|
+
findings = [finding for result in results for finding in result.findings]
|
|
139
|
+
annotate_git_exposure(root, findings)
|
|
140
|
+
findings.sort(key=finding_sort_key)
|
|
141
|
+
status = overall_status(results)
|
|
142
|
+
counts = severity_counts(findings)
|
|
143
|
+
privacy = []
|
|
144
|
+
for name in requested:
|
|
145
|
+
adapter = get_scanner(name)
|
|
146
|
+
privacy.append({
|
|
147
|
+
"tool": name,
|
|
148
|
+
"transmits_project_metadata": adapter.transmits_project_metadata,
|
|
149
|
+
"downloads_data": adapter.downloads_data,
|
|
150
|
+
"note": adapter.network_note,
|
|
151
|
+
})
|
|
152
|
+
return {
|
|
153
|
+
"schema_version": SCHEMA_VERSION,
|
|
154
|
+
# Microseconds keep ids strictly ordered even for scans started within the same second.
|
|
155
|
+
"scan_id": started.strftime("%Y%m%dT%H%M%S%fZ") + "-" + uuid.uuid4().hex[:6],
|
|
156
|
+
"uds_version": __version__,
|
|
157
|
+
"started_at": started.isoformat(),
|
|
158
|
+
"finished_at": datetime.now(timezone.utc).isoformat(),
|
|
159
|
+
"target": str(root),
|
|
160
|
+
"project": {
|
|
161
|
+
"detected": project["detected"],
|
|
162
|
+
"files_inventoried": project["files_inventoried"],
|
|
163
|
+
"inventory_truncated": project["inventory_truncated"],
|
|
164
|
+
},
|
|
165
|
+
"options": {"tools": requested, "allow_network": options.allow_network, "fail_on": config.fail_on,
|
|
166
|
+
"scope_fingerprint": scope_fingerprint(config.exclude_paths)},
|
|
167
|
+
"overall_status": status,
|
|
168
|
+
"scanners": [{k: v for k, v in r.to_dict().items() if k != "findings"} for r in results],
|
|
169
|
+
"summary": {
|
|
170
|
+
"finding_count": len(findings),
|
|
171
|
+
"by_severity": counts,
|
|
172
|
+
"by_tool": {r.tool: r.finding_count for r in results},
|
|
173
|
+
"by_category": {c: sum(1 for f in findings if f.category.value == c)
|
|
174
|
+
for c in ("secret", "code", "dependency", "configuration")},
|
|
175
|
+
"security_score": None,
|
|
176
|
+
"score_note": "Not calculated: UDS does not publish a numeric score without a validated, documented methodology.",
|
|
177
|
+
},
|
|
178
|
+
"coverage": {
|
|
179
|
+
"executed": [r.tool for r in results if r.status.executed],
|
|
180
|
+
"not_executed": [{"tool": r.tool, "status": r.status.value, "reason": r.reason} for r in results
|
|
181
|
+
if not r.status.executed],
|
|
182
|
+
},
|
|
183
|
+
"privacy": privacy,
|
|
184
|
+
"findings": [f.to_dict() for f in findings],
|
|
185
|
+
"limitations": [
|
|
186
|
+
"This report covers only the scanners listed as executed.",
|
|
187
|
+
"A clean result is not proof that the project is secure.",
|
|
188
|
+
"Severity is shown only when supplied by the scanner or derived from a published scale (CVSS); "
|
|
189
|
+
"other findings are unclassified.",
|
|
190
|
+
"Secret values, matched text and raw scanner output are intentionally excluded.",
|
|
191
|
+
],
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def exit_code_for(scan: dict[str, Any], fail_on: str = "none", require_complete: bool = False) -> int:
|
|
196
|
+
"""0 = success, 1 = findings at/above threshold, 2 = scan error / nothing scanned / incomplete (if required)."""
|
|
197
|
+
status = scan["overall_status"]
|
|
198
|
+
if status in ("not_scanned", "error") or any(s["status"] == "error" for s in scan["scanners"]):
|
|
199
|
+
return 2
|
|
200
|
+
if require_complete and status == "incomplete":
|
|
201
|
+
return 2
|
|
202
|
+
findings = [Finding.from_dict(f) for f in scan["findings"]]
|
|
203
|
+
return 1 if threshold_reached(findings, fail_on) else 0
|