patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,96 @@
1
+ """Parity between two runs of the same prompt on the same reports.
2
+
3
+ This is the quantisation parity check (proposal §7, §11): the self-hosted 4-bit model
4
+ against the hosted model, before trusting backfill output. It answers three questions:
5
+
6
+ * **Reliability:** which reports each run failed to answer (dead letters).
7
+ * **Quality:** precision, recall and F1 of each run on the reports *both* answered, and the
8
+ F1 difference with a paired bootstrap that resamples reports.
9
+ * **Agreement:** how many answers are identical, and where they differ, whether one run's
10
+ set contains the other's. A run that only ever adds symbols is more inclusive, which costs
11
+ precision but never recall.
12
+ """
13
+
14
+ import random
15
+ from collections.abc import Mapping, Set
16
+
17
+ from pydantic import BaseModel, ConfigDict
18
+
19
+ from patchsight.eval.metrics import Interval, SetScores, _counts, _percentile, _prf, score
20
+
21
+ # Outcome statuses that count as a valid answer. Anything else (dead letter, no candidates)
22
+ # leaves the report out of the paired comparison.
23
+ VALID = frozenset({"ok", "repaired"})
24
+ BOOTSTRAP = 5000
25
+ SEED = 20261116
26
+
27
+ # A run's answers: report ID -> vulnerable set, or None when the run has no valid answer.
28
+ Answers = Mapping[str, Set[str] | None]
29
+
30
+
31
+ class ParityScores(BaseModel):
32
+ model_config = ConfigDict(frozen=True)
33
+
34
+ reports: int
35
+ reference_failed: tuple[str, ...]
36
+ candidate_failed: tuple[str, ...]
37
+ paired: int
38
+ reference: SetScores
39
+ candidate: SetScores
40
+ f1_difference: float # reference minus candidate
41
+ f1_difference_ci95: Interval
42
+ identical: int
43
+ candidate_adds_only: int # differing reports where the candidate's set strictly contains
44
+ candidate_drops_only: int # differing reports where the candidate's set is strictly inside
45
+ both_add_and_drop: int
46
+ differing: tuple[str, ...]
47
+
48
+
49
+ def parity(
50
+ reference: Answers,
51
+ candidate: Answers,
52
+ truth: Mapping[str, Set[str]],
53
+ bootstrap: int = BOOTSTRAP,
54
+ seed: int = SEED,
55
+ ) -> ParityScores:
56
+ """Compare two runs over the reports in ``truth``."""
57
+ reports = sorted(truth)
58
+ ref_failed = tuple(r for r in reports if reference.get(r) is None)
59
+ cand_failed = tuple(r for r in reports if candidate.get(r) is None)
60
+ paired = [r for r in reports if reference.get(r) is not None and candidate.get(r) is not None]
61
+ ref = {r: set(reference[r] or ()) for r in paired}
62
+ cand = {r: set(candidate[r] or ()) for r in paired}
63
+ paired_truth = {r: truth[r] for r in paired}
64
+
65
+ ref_counts = [_counts(ref[r], truth[r]) for r in paired]
66
+ cand_counts = [_counts(cand[r], truth[r]) for r in paired]
67
+
68
+ def f1(indices: list[int]) -> float:
69
+ a = _prf(*(sum(getattr(ref_counts[i], k) for i in indices) for k in ("tp", "fp", "fn")))
70
+ b = _prf(*(sum(getattr(cand_counts[i], k) for i in indices) for k in ("tp", "fp", "fn")))
71
+ return a[2] - b[2]
72
+
73
+ everything = list(range(len(paired)))
74
+ rng = random.Random(seed) # noqa: S311 - statistical resampling, not security
75
+ draws = [
76
+ f1([rng.randrange(len(paired)) for _ in paired]) for _ in range(bootstrap if paired else 0)
77
+ ]
78
+ differing = [r for r in paired if ref[r] != cand[r]]
79
+ return ParityScores(
80
+ reports=len(reports),
81
+ reference_failed=ref_failed,
82
+ candidate_failed=cand_failed,
83
+ paired=len(paired),
84
+ reference=score(ref, paired_truth),
85
+ candidate=score(cand, paired_truth),
86
+ f1_difference=f1(everything) if paired else 0.0,
87
+ f1_difference_ci95=Interval(
88
+ low=_percentile(draws, 0.025) if draws else 0.0,
89
+ high=_percentile(draws, 0.975) if draws else 0.0,
90
+ ),
91
+ identical=len(paired) - len(differing),
92
+ candidate_adds_only=sum(cand[r] > ref[r] for r in differing),
93
+ candidate_drops_only=sum(cand[r] < ref[r] for r in differing),
94
+ both_add_and_drop=sum(not (cand[r] > ref[r] or cand[r] < ref[r]) for r in differing),
95
+ differing=tuple(differing),
96
+ )
@@ -0,0 +1,54 @@
1
+ """A product-relevant safety view of extraction results.
2
+
3
+ The benchmark metric (``eval.metrics``) compares names exactly and ignores entry points,
4
+ which keeps it comparable with Dunlap et al. The scanner behaves differently, in two ways:
5
+
6
+ * it matches **entry points** as well as the vulnerable functions themselves;
7
+ * the ground truth names functions inconsistently. Older GoVulnDB entries list
8
+ ``decryptTicket`` for the method ``Conn.decryptTicket``, and models write package
9
+ prefixes (``syscall.ForkExec``).
10
+
11
+ This view therefore counts *product symbols* (the vulnerable functions plus the validated
12
+ entry points) and matches names leniently, on a dotted boundary: ``a.b.c`` matches ``b.c``
13
+ and ``c``. Its miss rate answers the safety question: how often would the database hold
14
+ nothing that reaches the true vulnerable symbol?
15
+ """
16
+
17
+ from collections.abc import Mapping, Set
18
+
19
+ from pydantic import BaseModel, ConfigDict
20
+
21
+
22
+ def lenient_match(predicted: str, truth: str) -> bool:
23
+ return predicted == truth or predicted.endswith(f".{truth}") or truth.endswith(f".{predicted}")
24
+
25
+
26
+ class SafetyScores(BaseModel):
27
+ model_config = ConfigDict(frozen=True)
28
+
29
+ reports: int
30
+ truth_symbols: int
31
+ truth_found: int
32
+ recall: float
33
+ miss_rate: float
34
+
35
+
36
+ def safety(predicted: Mapping[str, Set[str]], truth: Mapping[str, Set[str]]) -> SafetyScores:
37
+ """Recall over true symbols and the report-level miss rate, with lenient matching."""
38
+ found = total = missed = with_truth = 0
39
+ for report, symbols in truth.items():
40
+ if not symbols:
41
+ continue
42
+ with_truth += 1
43
+ guesses = predicted.get(report, set())
44
+ hits = sum(any(lenient_match(p, t) for p in guesses) for t in symbols)
45
+ found += hits
46
+ total += len(symbols)
47
+ missed += hits == 0
48
+ return SafetyScores(
49
+ reports=with_truth,
50
+ truth_symbols=total,
51
+ truth_found=found,
52
+ recall=found / total if total else 0.0,
53
+ miss_rate=missed / with_truth if with_truth else 0.0,
54
+ )
patchsight/names.py ADDED
@@ -0,0 +1,76 @@
1
+ """Canonical names shared by the offline pipeline and the scan path.
2
+
3
+ Both halves must use these functions, so that a package or symbol name written into the
4
+ database always matches the name the scanner looks up.
5
+ """
6
+
7
+ from packaging.utils import canonicalize_name
8
+
9
+
10
+ def canonical_package_name(name: str) -> str:
11
+ """Return the PEP 503 normalised form of a distribution name.
12
+
13
+ >>> canonical_package_name("PyYAML")
14
+ 'pyyaml'
15
+ >>> canonical_package_name("zope.interface")
16
+ 'zope-interface'
17
+ """
18
+ return str(canonicalize_name(name.strip()))
19
+
20
+
21
+ _SOURCE_PREFIXES = frozenset({"lib", "lib3", "python", "py", "Lib"})
22
+ _PYTHON_SUFFIXES = (".py", ".pyi", ".pyx")
23
+
24
+ # Distributions whose import name differs from their distribution name. Used only as a
25
+ # hint alongside the import roots recorded in the database; an unknown mapping makes a
26
+ # verdict UNKNOWN, never NOT_IMPORTED.
27
+ KNOWN_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
28
+ "pyyaml": ("yaml",),
29
+ "pillow": ("PIL",),
30
+ "beautifulsoup4": ("bs4",),
31
+ "scikit-learn": ("sklearn",),
32
+ "python-dateutil": ("dateutil",),
33
+ "pyjwt": ("jwt",),
34
+ "pycryptodome": ("Crypto",),
35
+ "pycryptodomex": ("Cryptodome",),
36
+ "opencv-python": ("cv2",),
37
+ "protobuf": ("google.protobuf",),
38
+ "setuptools": ("setuptools", "pkg_resources"),
39
+ "msgpack-python": ("msgpack",),
40
+ "python-multipart": ("multipart", "python_multipart"),
41
+ "pymongo": ("pymongo", "bson", "gridfs"),
42
+ "attrs": ("attr", "attrs"),
43
+ }
44
+
45
+
46
+ def module_for_path(path: str) -> str | None:
47
+ """The importable module a repository path defines, or None if it cannot be told.
48
+
49
+ >>> module_for_path("lib/yaml/constructor.py")
50
+ 'yaml.constructor'
51
+ >>> module_for_path("airflow-core/src/airflow/models/dag.py")
52
+ 'airflow.models.dag'
53
+ >>> module_for_path("src/flask/__init__.py")
54
+ 'flask'
55
+ """
56
+ if not path.endswith(_PYTHON_SUFFIXES):
57
+ return None
58
+ parts = path.strip("/").rsplit(".", 1)[0].split("/")
59
+ if "src" in parts[:-1]:
60
+ parts = parts[len(parts) - 1 - parts[::-1].index("src") + 1 :]
61
+ else:
62
+ while len(parts) > 1 and (parts[0] in _SOURCE_PREFIXES or not parts[0].isidentifier()):
63
+ parts = parts[1:]
64
+ if parts and parts[-1] == "__init__":
65
+ parts = parts[:-1]
66
+ if not parts or not all(p.isidentifier() for p in parts):
67
+ return None
68
+ return ".".join(parts)
69
+
70
+
71
+ def import_names(package: str, roots: tuple[str, ...] = ()) -> tuple[str, ...]:
72
+ """Names a distribution may be imported as: recorded roots, known exceptions, default."""
73
+ canonical = canonical_package_name(package)
74
+ names = list(roots) + list(KNOWN_IMPORT_NAMES.get(canonical, ()))
75
+ names.append(canonical.replace("-", "_"))
76
+ return tuple(dict.fromkeys(names))
@@ -0,0 +1,4 @@
1
+ """Offline pipeline: ingestion, diff mapping, LLM extraction, database build.
2
+
3
+ This is the only package allowed to use an LLM. Nothing on the scan path may import it.
4
+ """