patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Parity between two runs of the same prompt on the same reports.
|
|
2
|
+
|
|
3
|
+
This is the quantisation parity check (proposal §7, §11): the self-hosted 4-bit model
|
|
4
|
+
against the hosted model, before trusting backfill output. It answers three questions:
|
|
5
|
+
|
|
6
|
+
* **Reliability:** which reports each run failed to answer (dead letters).
|
|
7
|
+
* **Quality:** precision, recall and F1 of each run on the reports *both* answered, and the
|
|
8
|
+
F1 difference with a paired bootstrap that resamples reports.
|
|
9
|
+
* **Agreement:** how many answers are identical, and where they differ, whether one run's
|
|
10
|
+
set contains the other's. A run that only ever adds symbols is more inclusive, which costs
|
|
11
|
+
precision but never recall.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import random
|
|
15
|
+
from collections.abc import Mapping, Set
|
|
16
|
+
|
|
17
|
+
from pydantic import BaseModel, ConfigDict
|
|
18
|
+
|
|
19
|
+
from patchsight.eval.metrics import Interval, SetScores, _counts, _percentile, _prf, score
|
|
20
|
+
|
|
21
|
+
# Outcome statuses that count as a valid answer. Anything else (dead letter, no candidates)
|
|
22
|
+
# leaves the report out of the paired comparison.
|
|
23
|
+
VALID = frozenset({"ok", "repaired"})
|
|
24
|
+
BOOTSTRAP = 5000
|
|
25
|
+
SEED = 20261116
|
|
26
|
+
|
|
27
|
+
# A run's answers: report ID -> vulnerable set, or None when the run has no valid answer.
|
|
28
|
+
Answers = Mapping[str, Set[str] | None]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ParityScores(BaseModel):
|
|
32
|
+
model_config = ConfigDict(frozen=True)
|
|
33
|
+
|
|
34
|
+
reports: int
|
|
35
|
+
reference_failed: tuple[str, ...]
|
|
36
|
+
candidate_failed: tuple[str, ...]
|
|
37
|
+
paired: int
|
|
38
|
+
reference: SetScores
|
|
39
|
+
candidate: SetScores
|
|
40
|
+
f1_difference: float # reference minus candidate
|
|
41
|
+
f1_difference_ci95: Interval
|
|
42
|
+
identical: int
|
|
43
|
+
candidate_adds_only: int # differing reports where the candidate's set strictly contains
|
|
44
|
+
candidate_drops_only: int # differing reports where the candidate's set is strictly inside
|
|
45
|
+
both_add_and_drop: int
|
|
46
|
+
differing: tuple[str, ...]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def parity(
|
|
50
|
+
reference: Answers,
|
|
51
|
+
candidate: Answers,
|
|
52
|
+
truth: Mapping[str, Set[str]],
|
|
53
|
+
bootstrap: int = BOOTSTRAP,
|
|
54
|
+
seed: int = SEED,
|
|
55
|
+
) -> ParityScores:
|
|
56
|
+
"""Compare two runs over the reports in ``truth``."""
|
|
57
|
+
reports = sorted(truth)
|
|
58
|
+
ref_failed = tuple(r for r in reports if reference.get(r) is None)
|
|
59
|
+
cand_failed = tuple(r for r in reports if candidate.get(r) is None)
|
|
60
|
+
paired = [r for r in reports if reference.get(r) is not None and candidate.get(r) is not None]
|
|
61
|
+
ref = {r: set(reference[r] or ()) for r in paired}
|
|
62
|
+
cand = {r: set(candidate[r] or ()) for r in paired}
|
|
63
|
+
paired_truth = {r: truth[r] for r in paired}
|
|
64
|
+
|
|
65
|
+
ref_counts = [_counts(ref[r], truth[r]) for r in paired]
|
|
66
|
+
cand_counts = [_counts(cand[r], truth[r]) for r in paired]
|
|
67
|
+
|
|
68
|
+
def f1(indices: list[int]) -> float:
|
|
69
|
+
a = _prf(*(sum(getattr(ref_counts[i], k) for i in indices) for k in ("tp", "fp", "fn")))
|
|
70
|
+
b = _prf(*(sum(getattr(cand_counts[i], k) for i in indices) for k in ("tp", "fp", "fn")))
|
|
71
|
+
return a[2] - b[2]
|
|
72
|
+
|
|
73
|
+
everything = list(range(len(paired)))
|
|
74
|
+
rng = random.Random(seed) # noqa: S311 - statistical resampling, not security
|
|
75
|
+
draws = [
|
|
76
|
+
f1([rng.randrange(len(paired)) for _ in paired]) for _ in range(bootstrap if paired else 0)
|
|
77
|
+
]
|
|
78
|
+
differing = [r for r in paired if ref[r] != cand[r]]
|
|
79
|
+
return ParityScores(
|
|
80
|
+
reports=len(reports),
|
|
81
|
+
reference_failed=ref_failed,
|
|
82
|
+
candidate_failed=cand_failed,
|
|
83
|
+
paired=len(paired),
|
|
84
|
+
reference=score(ref, paired_truth),
|
|
85
|
+
candidate=score(cand, paired_truth),
|
|
86
|
+
f1_difference=f1(everything) if paired else 0.0,
|
|
87
|
+
f1_difference_ci95=Interval(
|
|
88
|
+
low=_percentile(draws, 0.025) if draws else 0.0,
|
|
89
|
+
high=_percentile(draws, 0.975) if draws else 0.0,
|
|
90
|
+
),
|
|
91
|
+
identical=len(paired) - len(differing),
|
|
92
|
+
candidate_adds_only=sum(cand[r] > ref[r] for r in differing),
|
|
93
|
+
candidate_drops_only=sum(cand[r] < ref[r] for r in differing),
|
|
94
|
+
both_add_and_drop=sum(not (cand[r] > ref[r] or cand[r] < ref[r]) for r in differing),
|
|
95
|
+
differing=tuple(differing),
|
|
96
|
+
)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""A product-relevant safety view of extraction results.
|
|
2
|
+
|
|
3
|
+
The benchmark metric (``eval.metrics``) compares names exactly and ignores entry points,
|
|
4
|
+
which keeps it comparable with Dunlap et al. The scanner behaves differently, in two ways:
|
|
5
|
+
|
|
6
|
+
* it matches **entry points** as well as the vulnerable functions themselves;
|
|
7
|
+
* the ground truth names functions inconsistently. Older GoVulnDB entries list
|
|
8
|
+
``decryptTicket`` for the method ``Conn.decryptTicket``, and models write package
|
|
9
|
+
prefixes (``syscall.ForkExec``).
|
|
10
|
+
|
|
11
|
+
This view therefore counts *product symbols* (the vulnerable functions plus the validated
|
|
12
|
+
entry points) and matches names leniently, on a dotted boundary: ``a.b.c`` matches ``b.c``
|
|
13
|
+
and ``c``. Its miss rate answers the safety question: how often would the database hold
|
|
14
|
+
nothing that reaches the true vulnerable symbol?
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from collections.abc import Mapping, Set
|
|
18
|
+
|
|
19
|
+
from pydantic import BaseModel, ConfigDict
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def lenient_match(predicted: str, truth: str) -> bool:
|
|
23
|
+
return predicted == truth or predicted.endswith(f".{truth}") or truth.endswith(f".{predicted}")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SafetyScores(BaseModel):
|
|
27
|
+
model_config = ConfigDict(frozen=True)
|
|
28
|
+
|
|
29
|
+
reports: int
|
|
30
|
+
truth_symbols: int
|
|
31
|
+
truth_found: int
|
|
32
|
+
recall: float
|
|
33
|
+
miss_rate: float
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def safety(predicted: Mapping[str, Set[str]], truth: Mapping[str, Set[str]]) -> SafetyScores:
|
|
37
|
+
"""Recall over true symbols and the report-level miss rate, with lenient matching."""
|
|
38
|
+
found = total = missed = with_truth = 0
|
|
39
|
+
for report, symbols in truth.items():
|
|
40
|
+
if not symbols:
|
|
41
|
+
continue
|
|
42
|
+
with_truth += 1
|
|
43
|
+
guesses = predicted.get(report, set())
|
|
44
|
+
hits = sum(any(lenient_match(p, t) for p in guesses) for t in symbols)
|
|
45
|
+
found += hits
|
|
46
|
+
total += len(symbols)
|
|
47
|
+
missed += hits == 0
|
|
48
|
+
return SafetyScores(
|
|
49
|
+
reports=with_truth,
|
|
50
|
+
truth_symbols=total,
|
|
51
|
+
truth_found=found,
|
|
52
|
+
recall=found / total if total else 0.0,
|
|
53
|
+
miss_rate=missed / with_truth if with_truth else 0.0,
|
|
54
|
+
)
|
patchsight/names.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Canonical names shared by the offline pipeline and the scan path.
|
|
2
|
+
|
|
3
|
+
Both halves must use these functions, so that a package or symbol name written into the
|
|
4
|
+
database always matches the name the scanner looks up.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from packaging.utils import canonicalize_name
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def canonical_package_name(name: str) -> str:
|
|
11
|
+
"""Return the PEP 503 normalised form of a distribution name.
|
|
12
|
+
|
|
13
|
+
>>> canonical_package_name("PyYAML")
|
|
14
|
+
'pyyaml'
|
|
15
|
+
>>> canonical_package_name("zope.interface")
|
|
16
|
+
'zope-interface'
|
|
17
|
+
"""
|
|
18
|
+
return str(canonicalize_name(name.strip()))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
_SOURCE_PREFIXES = frozenset({"lib", "lib3", "python", "py", "Lib"})
|
|
22
|
+
_PYTHON_SUFFIXES = (".py", ".pyi", ".pyx")
|
|
23
|
+
|
|
24
|
+
# Distributions whose import name differs from their distribution name. Used only as a
|
|
25
|
+
# hint alongside the import roots recorded in the database; an unknown mapping makes a
|
|
26
|
+
# verdict UNKNOWN, never NOT_IMPORTED.
|
|
27
|
+
KNOWN_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
|
|
28
|
+
"pyyaml": ("yaml",),
|
|
29
|
+
"pillow": ("PIL",),
|
|
30
|
+
"beautifulsoup4": ("bs4",),
|
|
31
|
+
"scikit-learn": ("sklearn",),
|
|
32
|
+
"python-dateutil": ("dateutil",),
|
|
33
|
+
"pyjwt": ("jwt",),
|
|
34
|
+
"pycryptodome": ("Crypto",),
|
|
35
|
+
"pycryptodomex": ("Cryptodome",),
|
|
36
|
+
"opencv-python": ("cv2",),
|
|
37
|
+
"protobuf": ("google.protobuf",),
|
|
38
|
+
"setuptools": ("setuptools", "pkg_resources"),
|
|
39
|
+
"msgpack-python": ("msgpack",),
|
|
40
|
+
"python-multipart": ("multipart", "python_multipart"),
|
|
41
|
+
"pymongo": ("pymongo", "bson", "gridfs"),
|
|
42
|
+
"attrs": ("attr", "attrs"),
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def module_for_path(path: str) -> str | None:
|
|
47
|
+
"""The importable module a repository path defines, or None if it cannot be told.
|
|
48
|
+
|
|
49
|
+
>>> module_for_path("lib/yaml/constructor.py")
|
|
50
|
+
'yaml.constructor'
|
|
51
|
+
>>> module_for_path("airflow-core/src/airflow/models/dag.py")
|
|
52
|
+
'airflow.models.dag'
|
|
53
|
+
>>> module_for_path("src/flask/__init__.py")
|
|
54
|
+
'flask'
|
|
55
|
+
"""
|
|
56
|
+
if not path.endswith(_PYTHON_SUFFIXES):
|
|
57
|
+
return None
|
|
58
|
+
parts = path.strip("/").rsplit(".", 1)[0].split("/")
|
|
59
|
+
if "src" in parts[:-1]:
|
|
60
|
+
parts = parts[len(parts) - 1 - parts[::-1].index("src") + 1 :]
|
|
61
|
+
else:
|
|
62
|
+
while len(parts) > 1 and (parts[0] in _SOURCE_PREFIXES or not parts[0].isidentifier()):
|
|
63
|
+
parts = parts[1:]
|
|
64
|
+
if parts and parts[-1] == "__init__":
|
|
65
|
+
parts = parts[:-1]
|
|
66
|
+
if not parts or not all(p.isidentifier() for p in parts):
|
|
67
|
+
return None
|
|
68
|
+
return ".".join(parts)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def import_names(package: str, roots: tuple[str, ...] = ()) -> tuple[str, ...]:
|
|
72
|
+
"""Names a distribution may be imported as: recorded roots, known exceptions, default."""
|
|
73
|
+
canonical = canonical_package_name(package)
|
|
74
|
+
names = list(roots) + list(KNOWN_IMPORT_NAMES.get(canonical, ()))
|
|
75
|
+
names.append(canonical.replace("-", "_"))
|
|
76
|
+
return tuple(dict.fromkeys(names))
|