timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
scanners/sca_scanner.py
ADDED
|
@@ -0,0 +1,667 @@
|
|
|
1
|
+
"""Software composition analysis -- manifests and lockfiles, stdlib only, fully offline.
|
|
2
|
+
|
|
3
|
+
Five manifest families are read line by line: ``requirements*.txt``, ``pyproject.toml``,
|
|
4
|
+
``poetry.lock``, ``package.json`` and ``package-lock.json``. Each yields dependencies carrying
|
|
5
|
+
the *interval* of versions its spec allows, not a guessed resolved version.
|
|
6
|
+
|
|
7
|
+
That interval is the whole zero-false-positive argument. `cve_index.matching_advisories` reports
|
|
8
|
+
an advisory only when every version inside the interval is vulnerable, so an unbounded spec
|
|
9
|
+
(``requests>=2.0``, ``*``, ``latest``, a VCS URL, an editable install) is quiet by construction:
|
|
10
|
+
it allows a patched release, and nothing has been proven. A pinned ``==5.3.1`` is an interval of
|
|
11
|
+
one version, which is why lockfiles and pinned requirements are the loud, high-signal case.
|
|
12
|
+
|
|
13
|
+
Reachability is a second, separate proof. A vulnerable pin that no scanned Python file imports is
|
|
14
|
+
dormant manifest noise, so it is labelled ``UNREACHABLE_MANIFEST_ONLY`` at ``LOW`` confidence and
|
|
15
|
+
the default table floor hides it; one that production code imports is ``REACHABLE`` at ``HIGH``.
|
|
16
|
+
The label never changes whether a finding is emitted -- only how loudly it is presented.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import re
|
|
23
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
24
|
+
|
|
25
|
+
from .cve_index import (
|
|
26
|
+
VersionKey,
|
|
27
|
+
compare_versions,
|
|
28
|
+
import_roots_for,
|
|
29
|
+
matching_advisories,
|
|
30
|
+
normalize_package_name,
|
|
31
|
+
parse_version,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = ["is_manifest_path", "audit_sca_files", "tag_reachability", "collect_components",
|
|
35
|
+
"pinned_version_text", "CATEGORY"]
|
|
36
|
+
|
|
37
|
+
CATEGORY = "SCA_VULNERABLE_DEPENDENCY"
|
|
38
|
+
UNBOUNDED: Tuple[Optional[VersionKey], Optional[VersionKey], bool] = (None, None, True)
|
|
39
|
+
SEVERITY_RANK = {"CRITICAL": 4, "HIGH": 3, "MEDIUM": 2, "LOW": 1}
|
|
40
|
+
FINAL_RANK = 4
|
|
41
|
+
|
|
42
|
+
# A dependency: which registry, its name, the spec as written, the line it sits on, and the
|
|
43
|
+
# interval that spec allows. `upper_inclusive` distinguishes <=1.2.3 from <1.2.3.
|
|
44
|
+
Dependency = Tuple[str, str, str, int, Optional[VersionKey], Optional[VersionKey], bool]
|
|
45
|
+
|
|
46
|
+
_REQ_NAME_RE = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\s*(\[[^\]]*\])?\s*(.*)$")
|
|
47
|
+
_PY_CLAUSE_RE = re.compile(r"^(===|==|!=|~=|<=|>=|<|>)\s*(.+)$")
|
|
48
|
+
_TOML_KEY_RE = re.compile(r'^\s*(?:"([^"]+)"|\'([^\']+)\'|([A-Za-z0-9_.-]+))\s*=\s*(.*)$')
|
|
49
|
+
_TOML_TABLE_RE = re.compile(r"^\s*\[\[?([^\]]+)\]?\]\s*$")
|
|
50
|
+
_POETRY_VERSION_RE = re.compile(r'version\s*=\s*"([^"]*)"')
|
|
51
|
+
_LOCK_NAME_RE = re.compile(r'^\s*name\s*=\s*"([^"]+)"')
|
|
52
|
+
_LOCK_VERSION_RE = re.compile(r'^\s*version\s*=\s*"([^"]+)"')
|
|
53
|
+
_NPM_PREFIXES = ("workspace:", "file:", "link:", "portal:", "git:", "git+", "github:",
|
|
54
|
+
"http:", "https:", "npm:", "patch:")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def is_manifest_path(path: str) -> bool:
|
|
58
|
+
name = str(path).replace("\\", "/").rsplit("/", 1)[-1].lower()
|
|
59
|
+
if name in ("pyproject.toml", "poetry.lock", "package.json", "package-lock.json"):
|
|
60
|
+
return True
|
|
61
|
+
return name.startswith("requirements") and name.endswith(".txt")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _manifest_kind(path: str) -> Optional[str]:
|
|
65
|
+
name = str(path).replace("\\", "/").rsplit("/", 1)[-1].lower()
|
|
66
|
+
if name == "pyproject.toml":
|
|
67
|
+
return "pyproject"
|
|
68
|
+
if name == "poetry.lock":
|
|
69
|
+
return "poetry_lock"
|
|
70
|
+
if name == "package-lock.json":
|
|
71
|
+
return "package_lock"
|
|
72
|
+
if name == "package.json":
|
|
73
|
+
return "package_json"
|
|
74
|
+
if name.startswith("requirements") and name.endswith(".txt"):
|
|
75
|
+
return "requirements"
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# ── version intervals ─────────────────────────────────────────────────────
|
|
80
|
+
def _padded(parts: Sequence[int]) -> VersionKey:
|
|
81
|
+
filled = tuple(parts) + (0,) * (3 - len(parts))
|
|
82
|
+
return (filled, FINAL_RANK, 0)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _bump(parts: Sequence[int], index: int) -> VersionKey:
|
|
86
|
+
"""The exclusive upper bound of a prefix: 1.2 -> 1.3.0, 1 -> 2.0.0."""
|
|
87
|
+
filled = list(parts) + [0] * (index + 1 - len(parts))
|
|
88
|
+
filled[index] += 1
|
|
89
|
+
for position in range(index + 1, len(filled)):
|
|
90
|
+
filled[position] = 0
|
|
91
|
+
return (tuple(filled), FINAL_RANK, 0)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _intersect(clauses: Sequence[Tuple[Optional[VersionKey], Optional[VersionKey], bool]]
|
|
95
|
+
) -> Tuple[Optional[VersionKey], Optional[VersionKey], bool]:
|
|
96
|
+
lower: Optional[VersionKey] = None
|
|
97
|
+
upper: Optional[VersionKey] = None
|
|
98
|
+
inclusive = True
|
|
99
|
+
for lo, hi, hi_inclusive in clauses:
|
|
100
|
+
if lo is not None and (lower is None or compare_versions(lo, lower) > 0):
|
|
101
|
+
lower = lo
|
|
102
|
+
if hi is None:
|
|
103
|
+
continue
|
|
104
|
+
if upper is None:
|
|
105
|
+
upper, inclusive = hi, hi_inclusive
|
|
106
|
+
continue
|
|
107
|
+
order = compare_versions(hi, upper)
|
|
108
|
+
if order < 0:
|
|
109
|
+
upper, inclusive = hi, hi_inclusive
|
|
110
|
+
elif order == 0:
|
|
111
|
+
inclusive = inclusive and hi_inclusive
|
|
112
|
+
return (lower, upper, inclusive)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _python_clause(clause: str) -> Optional[Tuple[Optional[VersionKey], Optional[VersionKey], bool]]:
|
|
116
|
+
match = _PY_CLAUSE_RE.match(clause)
|
|
117
|
+
if not match:
|
|
118
|
+
key = parse_version(clause)
|
|
119
|
+
return (key, key, True) if key is not None else None
|
|
120
|
+
operator, value = match.group(1), match.group(2).strip()
|
|
121
|
+
if operator == "!=":
|
|
122
|
+
# An exclusion removes one version; it never bounds the interval from either side.
|
|
123
|
+
return UNBOUNDED
|
|
124
|
+
if value.endswith(".*"):
|
|
125
|
+
if operator not in ("==", "==="):
|
|
126
|
+
return None
|
|
127
|
+
base = parse_version(value[:-2])
|
|
128
|
+
if base is None:
|
|
129
|
+
return None
|
|
130
|
+
return (base, _bump(base[0], len(base[0]) - 1), False)
|
|
131
|
+
key = parse_version(value)
|
|
132
|
+
if key is None:
|
|
133
|
+
return None
|
|
134
|
+
if operator in ("==", "==="):
|
|
135
|
+
return (key, key, True)
|
|
136
|
+
if operator == "<=":
|
|
137
|
+
return (None, key, True)
|
|
138
|
+
if operator == "<":
|
|
139
|
+
return (None, key, False)
|
|
140
|
+
if operator in (">=", ">"):
|
|
141
|
+
return (key, None, True)
|
|
142
|
+
if operator == "~=":
|
|
143
|
+
# PEP 440 compatible release: ~=X.Y.Z means >=X.Y.Z, ==X.Y.*
|
|
144
|
+
if len(key[0]) < 2:
|
|
145
|
+
return None
|
|
146
|
+
return (key, _bump(key[0], len(key[0]) - 2), False)
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _python_bounds(spec: str) -> Tuple[Optional[VersionKey], Optional[VersionKey], bool]:
|
|
151
|
+
text = str(spec).split(";", 1)[0].strip()
|
|
152
|
+
if not text or text == "*":
|
|
153
|
+
return UNBOUNDED
|
|
154
|
+
clauses = []
|
|
155
|
+
for clause in text.split(","):
|
|
156
|
+
clause = clause.strip()
|
|
157
|
+
if not clause:
|
|
158
|
+
continue
|
|
159
|
+
parsed = _python_clause(clause)
|
|
160
|
+
if parsed is None:
|
|
161
|
+
# One unrecognised clause makes the whole spec unresolvable. Widening it to "anything"
|
|
162
|
+
# would be a guess, and a guess is how a dependency scanner invents a finding.
|
|
163
|
+
return UNBOUNDED
|
|
164
|
+
clauses.append(parsed)
|
|
165
|
+
return _intersect(clauses) if clauses else UNBOUNDED
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _npm_parts(token: str) -> Optional[List[int]]:
|
|
169
|
+
base = token.split("-", 1)[0].split("+", 1)[0]
|
|
170
|
+
pieces = base.split(".")
|
|
171
|
+
if len(pieces) > 3:
|
|
172
|
+
return None
|
|
173
|
+
parts: List[int] = []
|
|
174
|
+
for piece in pieces:
|
|
175
|
+
if piece in ("x", "X", "*"):
|
|
176
|
+
break
|
|
177
|
+
if not piece.isdigit():
|
|
178
|
+
return None
|
|
179
|
+
parts.append(int(piece))
|
|
180
|
+
return parts or None
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _npm_comparator(token: str) -> Optional[Tuple[Optional[VersionKey], Optional[VersionKey], bool]]:
|
|
184
|
+
token = token.strip()
|
|
185
|
+
if not token or token in ("*", "x", "X", "latest"):
|
|
186
|
+
return UNBOUNDED
|
|
187
|
+
if token.startswith("^"):
|
|
188
|
+
parts = _npm_parts(token[1:])
|
|
189
|
+
if not parts:
|
|
190
|
+
return None
|
|
191
|
+
# Caret pins the leftmost non-zero component; an all-zero prefix pins the last one given.
|
|
192
|
+
index = next((i for i, part in enumerate(parts) if part != 0), len(parts) - 1)
|
|
193
|
+
return (parse_version(token[1:]), _bump(parts, index), False)
|
|
194
|
+
if token.startswith("~"):
|
|
195
|
+
value = token[1:].lstrip(">").strip()
|
|
196
|
+
parts = _npm_parts(value)
|
|
197
|
+
if not parts:
|
|
198
|
+
return None
|
|
199
|
+
return (parse_version(value), _bump(parts, 1 if len(parts) >= 2 else 0), False)
|
|
200
|
+
for operator in (">=", "<=", ">", "<", "="):
|
|
201
|
+
if not token.startswith(operator):
|
|
202
|
+
continue
|
|
203
|
+
value = token[len(operator):].strip()
|
|
204
|
+
key = parse_version(value)
|
|
205
|
+
if key is None:
|
|
206
|
+
return None
|
|
207
|
+
# npm reads a partial `<=1.2` as <1.3.0 and `=1.2` as 1.2.x. Treating either as the
|
|
208
|
+
# single version 1.2.0 would narrow the interval and could condemn a patched release.
|
|
209
|
+
if operator in ("<=", "=") and len(key[0]) < 3:
|
|
210
|
+
return None
|
|
211
|
+
if operator in (">=", ">"):
|
|
212
|
+
return (key, None, True)
|
|
213
|
+
if operator == "<=":
|
|
214
|
+
return (None, key, True)
|
|
215
|
+
if operator == "<":
|
|
216
|
+
return (None, key, False)
|
|
217
|
+
return (key, key, True)
|
|
218
|
+
parts = _npm_parts(token)
|
|
219
|
+
if not parts:
|
|
220
|
+
return None
|
|
221
|
+
if len(parts) == 3:
|
|
222
|
+
key = parse_version(token)
|
|
223
|
+
return (key, key, True) if key is not None else None
|
|
224
|
+
return (_padded(parts), _bump(parts, len(parts) - 1), False)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _npm_branch(text: str) -> Optional[Tuple[Optional[VersionKey], Optional[VersionKey], bool]]:
|
|
228
|
+
if not text:
|
|
229
|
+
return UNBOUNDED
|
|
230
|
+
hyphen = re.split(r"\s+-\s+", text)
|
|
231
|
+
if len(hyphen) == 2:
|
|
232
|
+
low = parse_version(hyphen[0].strip())
|
|
233
|
+
high_parts = _npm_parts(hyphen[1].strip())
|
|
234
|
+
if low is None or not high_parts:
|
|
235
|
+
return None
|
|
236
|
+
high = (parse_version(hyphen[1].strip()) if len(high_parts) == 3
|
|
237
|
+
else _bump(high_parts, len(high_parts) - 1))
|
|
238
|
+
return (low, high, len(high_parts) == 3)
|
|
239
|
+
clauses = []
|
|
240
|
+
for token in text.split():
|
|
241
|
+
parsed = _npm_comparator(token)
|
|
242
|
+
if parsed is None:
|
|
243
|
+
return None
|
|
244
|
+
clauses.append(parsed)
|
|
245
|
+
return _intersect(clauses) if clauses else UNBOUNDED
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _npm_bounds(spec: str) -> Tuple[Optional[VersionKey], Optional[VersionKey], bool]:
|
|
249
|
+
text = str(spec).strip()
|
|
250
|
+
if not text or text.startswith(_NPM_PREFIXES):
|
|
251
|
+
return UNBOUNDED
|
|
252
|
+
branches = []
|
|
253
|
+
for branch in text.split("||"):
|
|
254
|
+
parsed = _npm_branch(branch.strip())
|
|
255
|
+
if parsed is None:
|
|
256
|
+
return UNBOUNDED
|
|
257
|
+
branches.append(parsed)
|
|
258
|
+
if not branches:
|
|
259
|
+
return UNBOUNDED
|
|
260
|
+
# A union is bounded above only when every branch is, and its bound is the widest one.
|
|
261
|
+
lower = None
|
|
262
|
+
upper = None
|
|
263
|
+
inclusive = False
|
|
264
|
+
for lo, hi, hi_inclusive in branches:
|
|
265
|
+
if lo is not None and (lower is None or compare_versions(lo, lower) < 0):
|
|
266
|
+
lower = lo
|
|
267
|
+
if hi is None:
|
|
268
|
+
return (lower, None, True)
|
|
269
|
+
if upper is None:
|
|
270
|
+
upper, inclusive = hi, hi_inclusive
|
|
271
|
+
continue
|
|
272
|
+
order = compare_versions(hi, upper)
|
|
273
|
+
if order > 0:
|
|
274
|
+
upper, inclusive = hi, hi_inclusive
|
|
275
|
+
elif order == 0:
|
|
276
|
+
inclusive = inclusive or hi_inclusive
|
|
277
|
+
return (lower, upper, inclusive)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _exact(version: str) -> Tuple[Optional[VersionKey], Optional[VersionKey], bool]:
|
|
281
|
+
key = parse_version(version)
|
|
282
|
+
return (key, key, True) if key is not None else UNBOUNDED
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
# ── requirements*.txt ─────────────────────────────────────────────────────
|
|
286
|
+
def _requirements(source: str) -> List[Dependency]:
|
|
287
|
+
out: List[Dependency] = []
|
|
288
|
+
pending = ""
|
|
289
|
+
start = 0
|
|
290
|
+
for lineno, raw in enumerate(source.splitlines(), 1):
|
|
291
|
+
line = raw.strip()
|
|
292
|
+
if not pending and (not line or line.startswith("#")):
|
|
293
|
+
continue
|
|
294
|
+
if not pending:
|
|
295
|
+
start = lineno
|
|
296
|
+
if line.endswith("\\"):
|
|
297
|
+
pending += line[:-1].strip() + " "
|
|
298
|
+
continue
|
|
299
|
+
text = (pending + line).strip()
|
|
300
|
+
pending = ""
|
|
301
|
+
text = text.split(" #", 1)[0].split(" --", 1)[0].split(";", 1)[0].strip()
|
|
302
|
+
if not text or text.startswith(("-", ".", "/")) or "://" in text or "@" in text:
|
|
303
|
+
# -e / -r / --index-url, a local path, a direct URL reference: no pinned version.
|
|
304
|
+
continue
|
|
305
|
+
match = _REQ_NAME_RE.match(text)
|
|
306
|
+
if not match:
|
|
307
|
+
continue
|
|
308
|
+
name, spec = match.group(1), match.group(3).strip()
|
|
309
|
+
if spec.startswith("(") and spec.endswith(")"):
|
|
310
|
+
spec = spec[1:-1].strip()
|
|
311
|
+
if not spec:
|
|
312
|
+
continue
|
|
313
|
+
lower, upper, inclusive = _python_bounds(spec)
|
|
314
|
+
out.append(("pypi", normalize_package_name(name), spec, start, lower, upper, inclusive))
|
|
315
|
+
return out
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
# ── pyproject.toml ────────────────────────────────────────────────────────
|
|
319
|
+
def _strip_toml_comment(text: str) -> str:
|
|
320
|
+
quote = ""
|
|
321
|
+
for index, char in enumerate(text):
|
|
322
|
+
if quote:
|
|
323
|
+
if char == quote:
|
|
324
|
+
quote = ""
|
|
325
|
+
elif char in "'\"":
|
|
326
|
+
quote = char
|
|
327
|
+
elif char == "#" and (index == 0 or text[index - 1] in " \t"):
|
|
328
|
+
return text[:index].rstrip()
|
|
329
|
+
return text.rstrip()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _toml_string(value: str) -> Optional[str]:
|
|
333
|
+
value = value.strip()
|
|
334
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
|
|
335
|
+
return value[1:-1]
|
|
336
|
+
return None
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _pyproject(source: str) -> List[Dependency]:
|
|
340
|
+
"""Read the two dependency shapes the task names, keeping each entry's own line number.
|
|
341
|
+
|
|
342
|
+
A line reader rather than tomllib: tomllib arrives in 3.11 and this project supports 3.10,
|
|
343
|
+
and a parsed TOML document has thrown away the line numbers a finding has to point at.
|
|
344
|
+
"""
|
|
345
|
+
out: List[Dependency] = []
|
|
346
|
+
lines = source.splitlines()
|
|
347
|
+
table = ""
|
|
348
|
+
index = 0
|
|
349
|
+
while index < len(lines):
|
|
350
|
+
raw = _strip_toml_comment(lines[index])
|
|
351
|
+
lineno = index + 1
|
|
352
|
+
index += 1
|
|
353
|
+
header = _TOML_TABLE_RE.match(raw)
|
|
354
|
+
if header:
|
|
355
|
+
table = header.group(1).strip().strip('"').strip("'")
|
|
356
|
+
continue
|
|
357
|
+
entry = _TOML_KEY_RE.match(raw)
|
|
358
|
+
if not entry:
|
|
359
|
+
continue
|
|
360
|
+
key = entry.group(1) or entry.group(2) or entry.group(3)
|
|
361
|
+
value = entry.group(4).strip()
|
|
362
|
+
if table == "project" and key == "dependencies":
|
|
363
|
+
body = value
|
|
364
|
+
while body.count("[") > body.count("]") and index < len(lines):
|
|
365
|
+
body += " " + _strip_toml_comment(lines[index])
|
|
366
|
+
index += 1
|
|
367
|
+
for element in re.findall(r'"([^"]*)"|\'([^\']*)\'', body):
|
|
368
|
+
text = (element[0] or element[1]).strip()
|
|
369
|
+
if not text:
|
|
370
|
+
continue
|
|
371
|
+
# An array element is a quoted literal on its own line, not a JSON key: it has
|
|
372
|
+
# no colon, so the key locator below would never match it.
|
|
373
|
+
found = next((i for i, raw in enumerate(lines, 1)
|
|
374
|
+
if '"' + text + '"' in raw or "'" + text + "'" in raw), 1)
|
|
375
|
+
match = _REQ_NAME_RE.match(text)
|
|
376
|
+
if not match:
|
|
377
|
+
continue
|
|
378
|
+
spec = match.group(3).strip()
|
|
379
|
+
if not spec:
|
|
380
|
+
continue
|
|
381
|
+
lower, upper, inclusive = _python_bounds(spec)
|
|
382
|
+
out.append(("pypi", normalize_package_name(match.group(1)), spec,
|
|
383
|
+
found, lower, upper, inclusive))
|
|
384
|
+
elif table == "tool.poetry.dependencies" and key.lower() != "python":
|
|
385
|
+
body = value
|
|
386
|
+
while body.count("{") > body.count("}") and index < len(lines):
|
|
387
|
+
body += " " + _strip_toml_comment(lines[index])
|
|
388
|
+
index += 1
|
|
389
|
+
if body.startswith("{"):
|
|
390
|
+
version = _POETRY_VERSION_RE.search(body)
|
|
391
|
+
if not version:
|
|
392
|
+
continue
|
|
393
|
+
spec = version.group(1)
|
|
394
|
+
else:
|
|
395
|
+
spec = _toml_string(body)
|
|
396
|
+
if spec is None:
|
|
397
|
+
continue
|
|
398
|
+
lower, upper, inclusive = _python_bounds(spec)
|
|
399
|
+
out.append(("pypi", normalize_package_name(key), spec, lineno, lower, upper, inclusive))
|
|
400
|
+
return out
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
# ── poetry.lock ───────────────────────────────────────────────────────────
|
|
404
|
+
def _poetry_lock(source: str) -> List[Dependency]:
|
|
405
|
+
out: List[Dependency] = []
|
|
406
|
+
name: Optional[str] = None
|
|
407
|
+
version: Optional[str] = None
|
|
408
|
+
lineno = 1
|
|
409
|
+
|
|
410
|
+
def flush() -> None:
|
|
411
|
+
if not name or not version:
|
|
412
|
+
return
|
|
413
|
+
lower, upper, inclusive = _exact(version)
|
|
414
|
+
out.append(("pypi", normalize_package_name(name), version, lineno, lower, upper, inclusive))
|
|
415
|
+
|
|
416
|
+
for index, raw in enumerate(source.splitlines(), 1):
|
|
417
|
+
line = raw.strip()
|
|
418
|
+
if line == "[[package]]":
|
|
419
|
+
flush()
|
|
420
|
+
name = version = None
|
|
421
|
+
lineno = index
|
|
422
|
+
continue
|
|
423
|
+
hit = _LOCK_NAME_RE.match(line)
|
|
424
|
+
if hit:
|
|
425
|
+
name = hit.group(1)
|
|
426
|
+
lineno = index
|
|
427
|
+
continue
|
|
428
|
+
hit = _LOCK_VERSION_RE.match(line)
|
|
429
|
+
if hit:
|
|
430
|
+
version = hit.group(1)
|
|
431
|
+
flush()
|
|
432
|
+
return out
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
# ── package.json / package-lock.json ──────────────────────────────────────
|
|
436
|
+
def _locate(lines: Sequence[str], key: str, hint: Optional[str], cursor: int) -> int:
|
|
437
|
+
"""Line of a JSON key, disambiguated by a value that must sit at or just below it.
|
|
438
|
+
|
|
439
|
+
A lockfile repeats a bare package name at every nesting level, so the version is what tells
|
|
440
|
+
two entries apart. The cursor keeps successive lookups in document order.
|
|
441
|
+
"""
|
|
442
|
+
needle = '"' + key + '"'
|
|
443
|
+
for index in range(cursor, len(lines)):
|
|
444
|
+
text = lines[index]
|
|
445
|
+
if needle not in text or ":" not in text:
|
|
446
|
+
continue
|
|
447
|
+
if hint is None or hint in text:
|
|
448
|
+
return index + 1
|
|
449
|
+
if any(hint in ahead for ahead in lines[index + 1:index + 9]):
|
|
450
|
+
return index + 1
|
|
451
|
+
return cursor + 1 if cursor < len(lines) else 1
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _npm_dependency(name: str, spec: str, lineno: int) -> Optional[Dependency]:
|
|
455
|
+
normalized = normalize_package_name(name)
|
|
456
|
+
if not normalized:
|
|
457
|
+
return None
|
|
458
|
+
lower, upper, inclusive = _npm_bounds(spec)
|
|
459
|
+
return ("npm", normalized, spec, lineno, lower, upper, inclusive)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _package_json(source: str) -> List[Dependency]:
|
|
463
|
+
try:
|
|
464
|
+
document = json.loads(source)
|
|
465
|
+
except ValueError:
|
|
466
|
+
return []
|
|
467
|
+
if not isinstance(document, dict):
|
|
468
|
+
return []
|
|
469
|
+
lines = source.splitlines()
|
|
470
|
+
out: List[Dependency] = []
|
|
471
|
+
cursor = 0
|
|
472
|
+
for section in ("dependencies", "devDependencies"):
|
|
473
|
+
entries = document.get(section)
|
|
474
|
+
if not isinstance(entries, dict):
|
|
475
|
+
continue
|
|
476
|
+
for name, spec in entries.items():
|
|
477
|
+
if not isinstance(spec, str):
|
|
478
|
+
continue
|
|
479
|
+
lineno = _locate(lines, str(name), spec, cursor)
|
|
480
|
+
cursor = lineno
|
|
481
|
+
dependency = _npm_dependency(str(name), spec, lineno)
|
|
482
|
+
if dependency is not None:
|
|
483
|
+
out.append(dependency)
|
|
484
|
+
return out
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _package_lock(source: str) -> List[Dependency]:
|
|
488
|
+
try:
|
|
489
|
+
document = json.loads(source)
|
|
490
|
+
except ValueError:
|
|
491
|
+
return []
|
|
492
|
+
if not isinstance(document, dict):
|
|
493
|
+
return []
|
|
494
|
+
lines = source.splitlines()
|
|
495
|
+
out: List[Dependency] = []
|
|
496
|
+
cursor = 0
|
|
497
|
+
packages = document.get("packages")
|
|
498
|
+
if isinstance(packages, dict):
|
|
499
|
+
for key, entry in packages.items():
|
|
500
|
+
if not key or not isinstance(entry, dict) or entry.get("link"):
|
|
501
|
+
continue
|
|
502
|
+
version = entry.get("version")
|
|
503
|
+
if not isinstance(version, str):
|
|
504
|
+
continue
|
|
505
|
+
name = str(key).rsplit("node_modules/", 1)[-1]
|
|
506
|
+
lineno = _locate(lines, str(key), '"version": "%s"' % version, cursor)
|
|
507
|
+
if lineno <= cursor:
|
|
508
|
+
lineno = _locate(lines, str(key), version, cursor)
|
|
509
|
+
cursor = lineno
|
|
510
|
+
dependency = _npm_dependency(name, version, lineno)
|
|
511
|
+
if dependency is not None:
|
|
512
|
+
out.append(dependency)
|
|
513
|
+
return out
|
|
514
|
+
|
|
515
|
+
def walk(entries: Any, depth: int) -> None:
|
|
516
|
+
nonlocal cursor
|
|
517
|
+
if not isinstance(entries, dict) or depth > 12:
|
|
518
|
+
return
|
|
519
|
+
for name, entry in entries.items():
|
|
520
|
+
if not isinstance(entry, dict):
|
|
521
|
+
continue
|
|
522
|
+
version = entry.get("version")
|
|
523
|
+
if isinstance(version, str):
|
|
524
|
+
lineno = _locate(lines, str(name), '"version": "%s"' % version, cursor)
|
|
525
|
+
if lineno <= cursor:
|
|
526
|
+
lineno = _locate(lines, str(name), version, cursor)
|
|
527
|
+
cursor = lineno
|
|
528
|
+
dependency = _npm_dependency(str(name), version, lineno)
|
|
529
|
+
if dependency is not None:
|
|
530
|
+
out.append(dependency)
|
|
531
|
+
walk(entry.get("dependencies"), depth + 1)
|
|
532
|
+
|
|
533
|
+
walk(document.get("dependencies"), 0)
|
|
534
|
+
return out
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
# The readers inventory every declared dependency, including a spec with no ceiling. Whether a
|
|
538
|
+
# finding exists is `matching_advisories`' decision alone, so the audit and the SBOM share one parse.
|
|
539
|
+
_PARSERS = {
|
|
540
|
+
"requirements": _requirements,
|
|
541
|
+
"pyproject": _pyproject,
|
|
542
|
+
"poetry_lock": _poetry_lock,
|
|
543
|
+
"package_json": _package_json,
|
|
544
|
+
"package_lock": _package_lock,
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
_VERSION_TOKEN = re.compile(
|
|
548
|
+
r"\d+(?:\.\d+)*(?:[-.]?(?:a|b|c|rc|alpha|beta|pre|preview|dev|post|rev|r)[-.\d]*)?",
|
|
549
|
+
re.IGNORECASE)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def pinned_version_text(spec: str, lower: Optional[VersionKey], upper: Optional[VersionKey],
|
|
553
|
+
upper_inclusive: bool) -> Optional[str]:
|
|
554
|
+
"""The one version a spec allows, in its own spelling, or None when it allows a range.
|
|
555
|
+
|
|
556
|
+
An SBOM component may carry a version only when the manifest proves exactly one. `^4.17.20`
|
|
557
|
+
and `~=5.3.1` allow many, so they inventory as version-less coordinates rather than as a
|
|
558
|
+
guess at whichever release the resolver happened to pick.
|
|
559
|
+
"""
|
|
560
|
+
if lower is None or upper is None or not upper_inclusive:
|
|
561
|
+
return None
|
|
562
|
+
if compare_versions(lower, upper) != 0:
|
|
563
|
+
return None
|
|
564
|
+
match = _VERSION_TOKEN.search(spec)
|
|
565
|
+
return match.group(0) if match else None
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def collect_components(files: Dict[str, str]) -> List[Dict[str, Any]]:
|
|
569
|
+
"""Every declared dependency in the manifests, vulnerable or not, for the SBOM inventory."""
|
|
570
|
+
out: List[Dict[str, Any]] = []
|
|
571
|
+
for file_path in sorted(files):
|
|
572
|
+
kind = _manifest_kind(file_path)
|
|
573
|
+
if kind is None:
|
|
574
|
+
continue
|
|
575
|
+
for ecosystem, name, spec, lineno, lower, upper, inclusive in _PARSERS[kind](files[file_path]):
|
|
576
|
+
out.append({
|
|
577
|
+
"ecosystem": ecosystem,
|
|
578
|
+
"name": name,
|
|
579
|
+
"version": pinned_version_text(spec, lower, upper, inclusive),
|
|
580
|
+
"spec": spec,
|
|
581
|
+
"file": file_path,
|
|
582
|
+
"line": max(1, int(lineno)),
|
|
583
|
+
})
|
|
584
|
+
return out
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
# ── findings ──────────────────────────────────────────────────────────────
|
|
588
|
+
def _describe(finding: Dict[str, Any], evidence: Optional[Tuple[str, str, int]]) -> str:
|
|
589
|
+
cves = ", ".join(finding["cves"])
|
|
590
|
+
head = (f"{finding['cwe']}: {finding['package']} {finding['spec']} allows only versions below"
|
|
591
|
+
f" the fix for {cves} (fixed in {finding['fixed_version']}). {finding['summary']}")
|
|
592
|
+
if evidence is not None:
|
|
593
|
+
return head + (f" REACHABLE: production code imports it as {evidence[0]} at"
|
|
594
|
+
f" {evidence[1]}:{evidence[2]}, so the vulnerable release is actually"
|
|
595
|
+
f" loaded. Upgrade to {finding['package']}>={finding['fixed_version']}.")
|
|
596
|
+
return head + (" UNREACHABLE_MANIFEST_ONLY: no scanned Python file imports it, so this is a"
|
|
597
|
+
" dormant pin rather than a live exposure. It is hidden by the default table"
|
|
598
|
+
" confidence floor; pass --min-confidence low to list it.")
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def audit_sca_files(files: Dict[str, str]) -> List[Dict[str, Any]]:
|
|
602
|
+
"""Audit manifests against the offline index. Reachability starts unproven, therefore LOW."""
|
|
603
|
+
findings: List[Dict[str, Any]] = []
|
|
604
|
+
for file_path in sorted(files):
|
|
605
|
+
kind = _manifest_kind(file_path)
|
|
606
|
+
if kind is None:
|
|
607
|
+
continue
|
|
608
|
+
for ecosystem, name, spec, lineno, lower, upper, inclusive in _PARSERS[kind](files[file_path]):
|
|
609
|
+
matched = matching_advisories(ecosystem, name, lower, upper, inclusive)
|
|
610
|
+
if not matched:
|
|
611
|
+
continue
|
|
612
|
+
ordered = sorted(matched, key=lambda item: (-SEVERITY_RANK.get(item[2], 0), item[0]))
|
|
613
|
+
primary = ordered[0]
|
|
614
|
+
# One pin can carry several advisories with different fixes; the upgrade target is the
|
|
615
|
+
# highest of them, compared as versions rather than as tuples of unequal length.
|
|
616
|
+
target = primary
|
|
617
|
+
for item in ordered:
|
|
618
|
+
candidate, current = parse_version(item[3]), parse_version(target[3])
|
|
619
|
+
if candidate is not None and (current is None
|
|
620
|
+
or compare_versions(candidate, current) > 0):
|
|
621
|
+
target = item
|
|
622
|
+
finding: Dict[str, Any] = {
|
|
623
|
+
"file": file_path,
|
|
624
|
+
"line": max(1, int(lineno)),
|
|
625
|
+
"cwe": primary[1],
|
|
626
|
+
"severity": primary[2],
|
|
627
|
+
"category": CATEGORY,
|
|
628
|
+
"package": name,
|
|
629
|
+
"ecosystem": ecosystem,
|
|
630
|
+
"spec": spec,
|
|
631
|
+
"cve": primary[0],
|
|
632
|
+
"cves": sorted({item[0] for item in matched}),
|
|
633
|
+
"fixed_version": target[3],
|
|
634
|
+
"summary": primary[5],
|
|
635
|
+
"related_cwes": ["CWE-1395"],
|
|
636
|
+
"reachability": "UNREACHABLE_MANIFEST_ONLY",
|
|
637
|
+
"confidence": "LOW",
|
|
638
|
+
}
|
|
639
|
+
finding["message"] = _describe(finding, None)
|
|
640
|
+
findings.append(finding)
|
|
641
|
+
return sorted(findings, key=lambda item: (item["file"], item["line"], item["cwe"]))
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def tag_reachability(findings: List[Dict[str, Any]],
|
|
645
|
+
imported_roots: Dict[str, Tuple[str, int]]) -> List[Dict[str, Any]]:
|
|
646
|
+
"""Upgrade a finding to REACHABLE/HIGH when an actual import proves the dependency is loaded.
|
|
647
|
+
|
|
648
|
+
``imported_roots`` maps a top-level import root to the first (file, line) that imports it.
|
|
649
|
+
Only an upgrade ever happens here: without proof a finding keeps its conservative LOW label,
|
|
650
|
+
so the reachability pass can never manufacture confidence. An npm dependency has no Python
|
|
651
|
+
import proof available and therefore stays LOW.
|
|
652
|
+
"""
|
|
653
|
+
for finding in findings:
|
|
654
|
+
if finding.get("category") != CATEGORY or finding.get("ecosystem") != "pypi":
|
|
655
|
+
continue
|
|
656
|
+
evidence = None
|
|
657
|
+
for root in import_roots_for(finding["package"]):
|
|
658
|
+
hit = imported_roots.get(root.split(".", 1)[0])
|
|
659
|
+
if hit is not None:
|
|
660
|
+
evidence = (root, hit[0], hit[1])
|
|
661
|
+
break
|
|
662
|
+
if evidence is None:
|
|
663
|
+
continue
|
|
664
|
+
finding["reachability"] = "REACHABLE"
|
|
665
|
+
finding["confidence"] = "HIGH"
|
|
666
|
+
finding["message"] = _describe(finding, evidence)
|
|
667
|
+
return findings
|