pickle-sentry 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pickle_sentry-0.2.0/LICENSE +21 -0
- pickle_sentry-0.2.0/PKG-INFO +146 -0
- pickle_sentry-0.2.0/README.md +134 -0
- pickle_sentry-0.2.0/pyproject.toml +23 -0
- pickle_sentry-0.2.0/setup.cfg +4 -0
- pickle_sentry-0.2.0/src/pickle_sentry/__init__.py +3 -0
- pickle_sentry-0.2.0/src/pickle_sentry/cli.py +102 -0
- pickle_sentry-0.2.0/src/pickle_sentry/hub.py +88 -0
- pickle_sentry-0.2.0/src/pickle_sentry/inspector.py +184 -0
- pickle_sentry-0.2.0/src/pickle_sentry/model.py +48 -0
- pickle_sentry-0.2.0/src/pickle_sentry/report.py +55 -0
- pickle_sentry-0.2.0/src/pickle_sentry/scanner.py +144 -0
- pickle_sentry-0.2.0/src/pickle_sentry.egg-info/PKG-INFO +146 -0
- pickle_sentry-0.2.0/src/pickle_sentry.egg-info/SOURCES.txt +19 -0
- pickle_sentry-0.2.0/src/pickle_sentry.egg-info/dependency_links.txt +1 -0
- pickle_sentry-0.2.0/src/pickle_sentry.egg-info/entry_points.txt +2 -0
- pickle_sentry-0.2.0/src/pickle_sentry.egg-info/top_level.txt +1 -0
- pickle_sentry-0.2.0/tests/test_cli.py +43 -0
- pickle_sentry-0.2.0/tests/test_hub.py +59 -0
- pickle_sentry-0.2.0/tests/test_inspector.py +73 -0
- pickle_sentry-0.2.0/tests/test_scanner.py +54 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Navid Khan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pickle-sentry
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Scan code and dataset files for unsafe pickle deserialization — including the Hugging Face datasets pandas builder RCE (CVE-2026-57845). Never executes the files it inspects.
|
|
5
|
+
Author: Navid Khan
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: security,pickle,deserialization,huggingface,cve-2026-57845,scanner
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Dynamic: license-file
|
|
12
|
+
|
|
13
|
+
# pickle-sentry
|
|
14
|
+
|
|
15
|
+
Scan code and dataset files for unsafe pickle deserialization — including the
|
|
16
|
+
Hugging Face `datasets` pandas builder RCE, **CVE-2026-57845** (CVSS 8.8).
|
|
17
|
+
|
|
18
|
+
Pickle is code, not data. Any file, checkpoint, or dataset that passes through
|
|
19
|
+
a pickle loader executes whatever its author put in it. `pickle-sentry` finds
|
|
20
|
+
those paths in your code, and inspects pickle files by *disassembly only* —
|
|
21
|
+
**it never executes the files it inspects** (its test suite proves this with a
|
|
22
|
+
canary payload).
|
|
23
|
+
|
|
24
|
+
## Why now
|
|
25
|
+
|
|
26
|
+
CVE-2026-57845 (published 23 Oct 2026): Hugging Face `datasets` through 5.1.0 —
|
|
27
|
+
the latest release — routes `load_dataset("pandas", data_files=...)` to
|
|
28
|
+
`pandas.read_pickle()`. A crafted pickle runs arbitrary Python with the
|
|
29
|
+
victim's privileges the moment the dataset loads. The upstream fix
|
|
30
|
+
(huggingface/datasets#8879) disables the pandas builder, but no patched
|
|
31
|
+
release exists yet. Until one ships, the mitigation is: find every place you
|
|
32
|
+
load pickle-backed data, and migrate to Parquet / CSV / JSON / Arrow.
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install -e .
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Zero dependencies. Python 3.9+.
|
|
41
|
+
|
|
42
|
+
## Use
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pickle-sentry scan-path/ # text report, exit 1 on high+
|
|
46
|
+
pickle-sentry . --format json --output report.json
|
|
47
|
+
pickle-sentry . --format sarif --output report.sarif # for GitHub code scanning
|
|
48
|
+
pickle-sentry . --fail-on critical # CI: only fail on critical
|
|
49
|
+
pickle-sentry . --exclude /tests/ # production surface only
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### Scan a Hugging Face Hub repo (new in 0.2.0)
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pickle-sentry hub stanfordnlp/imdb # a dataset
|
|
56
|
+
pickle-sentry hub hf-internal-testing/tiny-random-bert --repo-type model
|
|
57
|
+
pickle-sentry hub user/repo --max-size-mb 16 --format json
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Lists the repo's files via the Hub API, downloads pickle-family files
|
|
61
|
+
(`.pkl`, `.pickle`, `.joblib`, `.pt`, `.pth`, and classic `pytorch_model.bin`)
|
|
62
|
+
and inspects them by disassembly — including the pickle stream inside torch
|
|
63
|
+
zip archives — and AST-scans any Python files in the repo. Files over the
|
|
64
|
+
size cap are flagged, not downloaded. Set `HF_TOKEN` for private repos.
|
|
65
|
+
Nothing downloaded is ever executed.
|
|
66
|
+
|
|
67
|
+
## What it catches
|
|
68
|
+
|
|
69
|
+
**In Python code (AST scan — parses, never imports your code):**
|
|
70
|
+
|
|
71
|
+
| Rule | Severity | What |
|
|
72
|
+
|---|---|---|
|
|
73
|
+
| `HF-DATASETS-PANDAS-BUILDER` | critical | `load_dataset("pandas", ...)` — CVE-2026-57845 |
|
|
74
|
+
| `UNSAFE-DESERIALIZATION` | high | `pickle.load/loads`, `dill`, `cloudpickle`, `marshal`, `shelve`, `joblib.load` |
|
|
75
|
+
| `PANDAS-READ-PICKLE` | high | `pandas.read_pickle(...)` |
|
|
76
|
+
| `TORCH-LOAD-WEIGHTS-ONLY-FALSE` | high | `torch.load(..., weights_only=False)` |
|
|
77
|
+
| `TORCH-LOAD-NO-WEIGHTS-ONLY` | medium | `torch.load(...)` with no explicit `weights_only=True` |
|
|
78
|
+
| `YAML-UNSAFE-LOAD` | medium | `yaml.load(...)` without a SafeLoader |
|
|
79
|
+
| `NUMPY-ALLOW-PICKLE` | high | `numpy.load(..., allow_pickle=True)` |
|
|
80
|
+
|
|
81
|
+
**In data files (disassembly via `pickletools`, never unpickled):**
|
|
82
|
+
|
|
83
|
+
| Rule | Severity | What |
|
|
84
|
+
|---|---|---|
|
|
85
|
+
| `PICKLE-DANGEROUS-REDUCE` | critical | Pickle calls `os.system`, `subprocess.*`, `eval`, `exec`, etc. via `REDUCE` |
|
|
86
|
+
| `PICKLE-REDUCE-CALLABLE` | high | Pickle invokes a non-standard callable via `REDUCE` (the CVE PoC payload's shape) |
|
|
87
|
+
| `PICKLE-DANGEROUS-GLOBAL` / `PICKLE-SUSPICIOUS-MODULE` | high/medium | Dangerous globals referenced without a reduce |
|
|
88
|
+
| `PICKLE-FILE-PRESENT` | medium | Any other `.pkl` / `.pickle` / `.joblib` file, and pickle streams inside torch `.pt`/`.pth` zip archives |
|
|
89
|
+
|
|
90
|
+
## Real-life use case
|
|
91
|
+
|
|
92
|
+
[docs/USE-CASE.md](docs/USE-CASE.md) walks through a real one end to end:
|
|
93
|
+
scanning the **actual published CVE-2026-57845 PoC repository** before
|
|
94
|
+
touching it. pickle-sentry flags the vulnerable loader (`run_poc.py:139`,
|
|
95
|
+
critical) and disassembles the live payload (`attacker-controlled.pkl`,
|
|
96
|
+
high) — and the PoC's own `marker.txt` proves the payload never executed
|
|
97
|
+
during the scan.
|
|
98
|
+
|
|
99
|
+
## Live examples
|
|
100
|
+
|
|
101
|
+
Scanned two real codebases on 9 Nov 2026 — full results, methodology, and
|
|
102
|
+
honest limits in [docs/LIVE-EXAMPLES.md](docs/LIVE-EXAMPLES.md):
|
|
103
|
+
|
|
104
|
+
- **huggingface/datasets**: 258 files, 24 findings — and exactly **one** in
|
|
105
|
+
production source: the CVE sink itself, `pd.read_pickle(f)` at
|
|
106
|
+
`packaged_modules/pandas/pandas.py:56`.
|
|
107
|
+
- **scikit-learn**: 1,037 files, 73 findings — 16 outside tests, concentrated
|
|
108
|
+
in `sklearn/datasets/` loaders unpickling cached copies of remotely fetched
|
|
109
|
+
datasets.
|
|
110
|
+
|
|
111
|
+
Also included: a composite **GitHub Action** (`action.yml`, use it as
|
|
112
|
+
`uses: Sun-Wukhan/pickle-sentry@main`) and a **pre-commit hook**
|
|
113
|
+
(`.pre-commit-hooks.yaml`). Use `--exclude /tests/` to cut test-fixture noise.
|
|
114
|
+
|
|
115
|
+
## Demo
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pickle-sentry examples/vulnerable # flags the CVE pattern + every sink, exit 1
|
|
119
|
+
pickle-sentry examples/safe # clean, exit 0
|
|
120
|
+
pickle-sentry examples/hf-datasets-pandas.py
|
|
121
|
+
# ^ the actual upstream datasets pandas builder (verbatim from
|
|
122
|
+
# huggingface/datasets main): flags pd.read_pickle(f) at line 56 —
|
|
123
|
+
# the exact sink behind CVE-2026-57845.
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## Migrate off pickle
|
|
127
|
+
|
|
128
|
+
- Datasets: `load_dataset("parquet" | "csv" | "json" | "arrow", ...)`
|
|
129
|
+
- Your own DataFrame: `Dataset.from_pandas(df)` instead of a pickle file
|
|
130
|
+
- DataFrames on disk: `df.to_parquet(...)` / `pd.read_parquet(...)`
|
|
131
|
+
- Torch: `torch.load(path, weights_only=True)`
|
|
132
|
+
|
|
133
|
+
## Tests
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
pip install pytest
|
|
137
|
+
pytest
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
The inspector tests build a real malicious pickle (`__reduce__` → `os.system`)
|
|
141
|
+
and assert two things: it is flagged critical, and the payload's canary file
|
|
142
|
+
**never appears** — i.e. inspection did not execute it.
|
|
143
|
+
|
|
144
|
+
## Licence
|
|
145
|
+
|
|
146
|
+
MIT
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# pickle-sentry
|
|
2
|
+
|
|
3
|
+
Scan code and dataset files for unsafe pickle deserialization — including the
|
|
4
|
+
Hugging Face `datasets` pandas builder RCE, **CVE-2026-57845** (CVSS 8.8).
|
|
5
|
+
|
|
6
|
+
Pickle is code, not data. Any file, checkpoint, or dataset that passes through
|
|
7
|
+
a pickle loader executes whatever its author put in it. `pickle-sentry` finds
|
|
8
|
+
those paths in your code, and inspects pickle files by *disassembly only* —
|
|
9
|
+
**it never executes the files it inspects** (its test suite proves this with a
|
|
10
|
+
canary payload).
|
|
11
|
+
|
|
12
|
+
## Why now
|
|
13
|
+
|
|
14
|
+
CVE-2026-57845 (published 23 Oct 2026): Hugging Face `datasets` through 5.1.0 —
|
|
15
|
+
the latest release — routes `load_dataset("pandas", data_files=...)` to
|
|
16
|
+
`pandas.read_pickle()`. A crafted pickle runs arbitrary Python with the
|
|
17
|
+
victim's privileges the moment the dataset loads. The upstream fix
|
|
18
|
+
(huggingface/datasets#8879) disables the pandas builder, but no patched
|
|
19
|
+
release exists yet. Until one ships, the mitigation is: find every place you
|
|
20
|
+
load pickle-backed data, and migrate to Parquet / CSV / JSON / Arrow.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install -e .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Zero dependencies. Python 3.9+.
|
|
29
|
+
|
|
30
|
+
## Use
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pickle-sentry scan-path/ # text report, exit 1 on high+
|
|
34
|
+
pickle-sentry . --format json --output report.json
|
|
35
|
+
pickle-sentry . --format sarif --output report.sarif # for GitHub code scanning
|
|
36
|
+
pickle-sentry . --fail-on critical # CI: only fail on critical
|
|
37
|
+
pickle-sentry . --exclude /tests/ # production surface only
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Scan a Hugging Face Hub repo (new in 0.2.0)
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pickle-sentry hub stanfordnlp/imdb # a dataset
|
|
44
|
+
pickle-sentry hub hf-internal-testing/tiny-random-bert --repo-type model
|
|
45
|
+
pickle-sentry hub user/repo --max-size-mb 16 --format json
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Lists the repo's files via the Hub API, downloads pickle-family files
|
|
49
|
+
(`.pkl`, `.pickle`, `.joblib`, `.pt`, `.pth`, and classic `pytorch_model.bin`)
|
|
50
|
+
and inspects them by disassembly — including the pickle stream inside torch
|
|
51
|
+
zip archives — and AST-scans any Python files in the repo. Files over the
|
|
52
|
+
size cap are flagged, not downloaded. Set `HF_TOKEN` for private repos.
|
|
53
|
+
Nothing downloaded is ever executed.
|
|
54
|
+
|
|
55
|
+
## What it catches
|
|
56
|
+
|
|
57
|
+
**In Python code (AST scan — parses, never imports your code):**
|
|
58
|
+
|
|
59
|
+
| Rule | Severity | What |
|
|
60
|
+
|---|---|---|
|
|
61
|
+
| `HF-DATASETS-PANDAS-BUILDER` | critical | `load_dataset("pandas", ...)` — CVE-2026-57845 |
|
|
62
|
+
| `UNSAFE-DESERIALIZATION` | high | `pickle.load/loads`, `dill`, `cloudpickle`, `marshal`, `shelve`, `joblib.load` |
|
|
63
|
+
| `PANDAS-READ-PICKLE` | high | `pandas.read_pickle(...)` |
|
|
64
|
+
| `TORCH-LOAD-WEIGHTS-ONLY-FALSE` | high | `torch.load(..., weights_only=False)` |
|
|
65
|
+
| `TORCH-LOAD-NO-WEIGHTS-ONLY` | medium | `torch.load(...)` with no explicit `weights_only=True` |
|
|
66
|
+
| `YAML-UNSAFE-LOAD` | medium | `yaml.load(...)` without a SafeLoader |
|
|
67
|
+
| `NUMPY-ALLOW-PICKLE` | high | `numpy.load(..., allow_pickle=True)` |
|
|
68
|
+
|
|
69
|
+
**In data files (disassembly via `pickletools`, never unpickled):**
|
|
70
|
+
|
|
71
|
+
| Rule | Severity | What |
|
|
72
|
+
|---|---|---|
|
|
73
|
+
| `PICKLE-DANGEROUS-REDUCE` | critical | Pickle calls `os.system`, `subprocess.*`, `eval`, `exec`, etc. via `REDUCE` |
|
|
74
|
+
| `PICKLE-REDUCE-CALLABLE` | high | Pickle invokes a non-standard callable via `REDUCE` (the CVE PoC payload's shape) |
|
|
75
|
+
| `PICKLE-DANGEROUS-GLOBAL` / `PICKLE-SUSPICIOUS-MODULE` | high/medium | Dangerous globals referenced without a reduce |
|
|
76
|
+
| `PICKLE-FILE-PRESENT` | medium | Any other `.pkl` / `.pickle` / `.joblib` file, and pickle streams inside torch `.pt`/`.pth` zip archives |
|
|
77
|
+
|
|
78
|
+
## Real-life use case
|
|
79
|
+
|
|
80
|
+
[docs/USE-CASE.md](docs/USE-CASE.md) walks through a real one end to end:
|
|
81
|
+
scanning the **actual published CVE-2026-57845 PoC repository** before
|
|
82
|
+
touching it. pickle-sentry flags the vulnerable loader (`run_poc.py:139`,
|
|
83
|
+
critical) and disassembles the live payload (`attacker-controlled.pkl`,
|
|
84
|
+
high) — and the PoC's own `marker.txt` proves the payload never executed
|
|
85
|
+
during the scan.
|
|
86
|
+
|
|
87
|
+
## Live examples
|
|
88
|
+
|
|
89
|
+
Scanned two real codebases on 9 Nov 2026 — full results, methodology, and
|
|
90
|
+
honest limits in [docs/LIVE-EXAMPLES.md](docs/LIVE-EXAMPLES.md):
|
|
91
|
+
|
|
92
|
+
- **huggingface/datasets**: 258 files, 24 findings — and exactly **one** in
|
|
93
|
+
production source: the CVE sink itself, `pd.read_pickle(f)` at
|
|
94
|
+
`packaged_modules/pandas/pandas.py:56`.
|
|
95
|
+
- **scikit-learn**: 1,037 files, 73 findings — 16 outside tests, concentrated
|
|
96
|
+
in `sklearn/datasets/` loaders unpickling cached copies of remotely fetched
|
|
97
|
+
datasets.
|
|
98
|
+
|
|
99
|
+
Also included: a composite **GitHub Action** (`action.yml`, use it as
|
|
100
|
+
`uses: Sun-Wukhan/pickle-sentry@main`) and a **pre-commit hook**
|
|
101
|
+
(`.pre-commit-hooks.yaml`). Use `--exclude /tests/` to cut test-fixture noise.
|
|
102
|
+
|
|
103
|
+
## Demo
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
pickle-sentry examples/vulnerable # flags the CVE pattern + every sink, exit 1
|
|
107
|
+
pickle-sentry examples/safe # clean, exit 0
|
|
108
|
+
pickle-sentry examples/hf-datasets-pandas.py
|
|
109
|
+
# ^ the actual upstream datasets pandas builder (verbatim from
|
|
110
|
+
# huggingface/datasets main): flags pd.read_pickle(f) at line 56 —
|
|
111
|
+
# the exact sink behind CVE-2026-57845.
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Migrate off pickle
|
|
115
|
+
|
|
116
|
+
- Datasets: `load_dataset("parquet" | "csv" | "json" | "arrow", ...)`
|
|
117
|
+
- Your own DataFrame: `Dataset.from_pandas(df)` instead of a pickle file
|
|
118
|
+
- DataFrames on disk: `df.to_parquet(...)` / `pd.read_parquet(...)`
|
|
119
|
+
- Torch: `torch.load(path, weights_only=True)`
|
|
120
|
+
|
|
121
|
+
## Tests
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
pip install pytest
|
|
125
|
+
pytest
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The inspector tests build a real malicious pickle (`__reduce__` → `os.system`)
|
|
129
|
+
and assert two things: it is flagged critical, and the payload's canary file
|
|
130
|
+
**never appears** — i.e. inspection did not execute it.
|
|
131
|
+
|
|
132
|
+
## Licence
|
|
133
|
+
|
|
134
|
+
MIT
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pickle-sentry"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Scan code and dataset files for unsafe pickle deserialization — including the Hugging Face datasets pandas builder RCE (CVE-2026-57845). Never executes the files it inspects."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Navid Khan" }]
|
|
13
|
+
keywords = ["security", "pickle", "deserialization", "huggingface", "cve-2026-57845", "scanner"]
|
|
14
|
+
dependencies = []
|
|
15
|
+
|
|
16
|
+
[project.scripts]
|
|
17
|
+
pickle-sentry = "pickle_sentry.cli:main"
|
|
18
|
+
|
|
19
|
+
[tool.setuptools.packages.find]
|
|
20
|
+
where = ["src"]
|
|
21
|
+
|
|
22
|
+
[tool.pytest.ini_options]
|
|
23
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from . import __version__
|
|
8
|
+
from .inspector import inspect_file
|
|
9
|
+
from .model import ScanResult
|
|
10
|
+
from .report import to_json, to_sarif, to_text
|
|
11
|
+
from .scanner import scan_python_file
|
|
12
|
+
|
|
13
|
+
SKIP_DIRS = {".git", "__pycache__", ".venv", "venv", "node_modules", ".tox", ".mypy_cache"}
|
|
14
|
+
PICKLE_EXTS = {".pkl", ".pickle", ".joblib", ".pt", ".pth"}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def collect(root: str, exclude: list[str] | None = None) -> ScanResult:
|
|
18
|
+
exclude = exclude or []
|
|
19
|
+
result = ScanResult(root=root)
|
|
20
|
+
paths: list[str] = []
|
|
21
|
+
if os.path.isfile(root):
|
|
22
|
+
paths = [root]
|
|
23
|
+
else:
|
|
24
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
25
|
+
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
|
|
26
|
+
for name in filenames:
|
|
27
|
+
paths.append(os.path.join(dirpath, name))
|
|
28
|
+
if exclude:
|
|
29
|
+
paths = [p for p in paths if not any(x in p for x in exclude)]
|
|
30
|
+
for p in sorted(paths):
|
|
31
|
+
ext = os.path.splitext(p)[1].lower()
|
|
32
|
+
if ext == ".py":
|
|
33
|
+
result.files_scanned += 1
|
|
34
|
+
result.findings.extend(scan_python_file(p))
|
|
35
|
+
elif ext in PICKLE_EXTS:
|
|
36
|
+
result.pickle_files_inspected += 1
|
|
37
|
+
result.findings.extend(inspect_file(p))
|
|
38
|
+
return result
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _hub_main(argv: list[str]) -> int:
|
|
42
|
+
from .hub import scan_hub_repo
|
|
43
|
+
ap = argparse.ArgumentParser(
|
|
44
|
+
prog="pickle-sentry hub",
|
|
45
|
+
description="Scan a Hugging Face Hub repository for pickle files and unsafe "
|
|
46
|
+
"deserialization. Files are inspected by disassembly, never executed.")
|
|
47
|
+
ap.add_argument("repo_id", help="e.g. user/dataset-name")
|
|
48
|
+
ap.add_argument("--repo-type", choices=["dataset", "model", "space"], default="dataset")
|
|
49
|
+
ap.add_argument("--revision", default="main")
|
|
50
|
+
ap.add_argument("--max-size-mb", type=float, default=64.0,
|
|
51
|
+
help="Per-file inspection cap in MB (default: 64)")
|
|
52
|
+
ap.add_argument("--format", choices=["text", "json", "sarif"], default="text")
|
|
53
|
+
ap.add_argument("--output", help="Write the report to this file instead of stdout")
|
|
54
|
+
ap.add_argument("--fail-on", choices=["critical", "high", "medium", "low", "none"],
|
|
55
|
+
default="high")
|
|
56
|
+
args = ap.parse_args(argv)
|
|
57
|
+
result = scan_hub_repo(args.repo_id, repo_type=args.repo_type, revision=args.revision,
|
|
58
|
+
max_bytes=int(args.max_size_mb * 1024 * 1024))
|
|
59
|
+
rendered = {"text": to_text, "json": to_json, "sarif": to_sarif}[args.format](result)
|
|
60
|
+
if args.output:
|
|
61
|
+
with open(args.output, "w", encoding="utf-8") as fh:
|
|
62
|
+
fh.write(rendered)
|
|
63
|
+
else:
|
|
64
|
+
sys.stdout.write(rendered)
|
|
65
|
+
return 1 if result.meets(args.fail_on) else 0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def main(argv: list[str] | None = None) -> int:
|
|
69
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
70
|
+
if argv and argv[0] == "hub":
|
|
71
|
+
return _hub_main(argv[1:])
|
|
72
|
+
ap = argparse.ArgumentParser(
|
|
73
|
+
prog="pickle-sentry",
|
|
74
|
+
description="Scan code and dataset files for unsafe pickle deserialization "
|
|
75
|
+
"(incl. CVE-2026-57845). Inspected pickle files are never executed.")
|
|
76
|
+
ap.add_argument("path", help="File or directory to scan")
|
|
77
|
+
ap.add_argument("--format", choices=["text", "json", "sarif"], default="text")
|
|
78
|
+
ap.add_argument("--output", help="Write the report to this file instead of stdout")
|
|
79
|
+
ap.add_argument("--fail-on", choices=["critical", "high", "medium", "low", "none"],
|
|
80
|
+
default="high",
|
|
81
|
+
help="Exit 1 if any finding is at or above this severity (default: high)")
|
|
82
|
+
ap.add_argument("--exclude", action="append", default=[],
|
|
83
|
+
help="Skip paths containing this substring (repeatable, e.g. --exclude /tests/)")
|
|
84
|
+
ap.add_argument("--version", action="version", version=f"pickle-sentry {__version__}")
|
|
85
|
+
args = ap.parse_args(argv)
|
|
86
|
+
|
|
87
|
+
if not os.path.exists(args.path):
|
|
88
|
+
print(f"error: no such path: {args.path}", file=sys.stderr)
|
|
89
|
+
return 2
|
|
90
|
+
|
|
91
|
+
result = collect(args.path, exclude=args.exclude)
|
|
92
|
+
rendered = {"text": to_text, "json": to_json, "sarif": to_sarif}[args.format](result)
|
|
93
|
+
if args.output:
|
|
94
|
+
with open(args.output, "w", encoding="utf-8") as fh:
|
|
95
|
+
fh.write(rendered)
|
|
96
|
+
else:
|
|
97
|
+
sys.stdout.write(rendered)
|
|
98
|
+
return 1 if result.meets(args.fail_on) else 0
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
if __name__ == "__main__":
|
|
102
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Scan a Hugging Face Hub repository without executing anything in it.
|
|
2
|
+
|
|
3
|
+
Lists the repo's files via the Hub API, downloads pickle-family files and
|
|
4
|
+
inspects them by disassembly only, and AST-scans any Python files.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import urllib.request
|
|
11
|
+
import urllib.error
|
|
12
|
+
|
|
13
|
+
from .inspector import inspect_pickle_bytes
|
|
14
|
+
from .model import Finding, ScanResult
|
|
15
|
+
from .scanner import scan_python_source
|
|
16
|
+
|
|
17
|
+
PICKLE_EXTS = {".pkl", ".pickle", ".joblib", ".pt", ".pth"}
|
|
18
|
+
# Model weights in classic PyTorch format are pickle zip archives.
|
|
19
|
+
MODEL_PICKLE_NAMES = {"pytorch_model.bin"}
|
|
20
|
+
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # per file
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _default_fetch(url: str) -> bytes:
|
|
24
|
+
req = urllib.request.Request(url, headers={"User-Agent": "pickle-sentry/0.2"})
|
|
25
|
+
token = os.environ.get("HF_TOKEN")
|
|
26
|
+
if token:
|
|
27
|
+
req.add_header("Authorization", f"Bearer {token}")
|
|
28
|
+
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
29
|
+
return resp.read()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _api_base(repo_type: str) -> str:
|
|
33
|
+
return {"dataset": "https://huggingface.co/api/datasets/",
|
|
34
|
+
"model": "https://huggingface.co/api/models/",
|
|
35
|
+
"space": "https://huggingface.co/api/spaces/"}[repo_type]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _resolve_base(repo_id: str, repo_type: str, revision: str) -> str:
|
|
39
|
+
prefix = {"dataset": "datasets/", "model": "", "space": "spaces/"}[repo_type]
|
|
40
|
+
return f"https://huggingface.co/{prefix}{repo_id}/resolve/{revision}/"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def scan_hub_repo(repo_id: str, repo_type: str = "dataset", revision: str = "main",
|
|
44
|
+
max_bytes: int = DEFAULT_MAX_BYTES, fetch=None) -> ScanResult:
|
|
45
|
+
fetch = fetch or _default_fetch
|
|
46
|
+
result = ScanResult(root=f"hub://{repo_type}/{repo_id}@{revision}")
|
|
47
|
+
try:
|
|
48
|
+
info = json.loads(fetch(_api_base(repo_type) + repo_id))
|
|
49
|
+
except (urllib.error.URLError, ValueError) as exc:
|
|
50
|
+
result.findings.append(Finding("HUB-UNREACHABLE", "low", result.root, None,
|
|
51
|
+
f"Could not read Hub repo info: {exc}"))
|
|
52
|
+
return result
|
|
53
|
+
|
|
54
|
+
siblings = info.get("siblings") or []
|
|
55
|
+
base = _resolve_base(repo_id, repo_type, revision)
|
|
56
|
+
for sib in siblings:
|
|
57
|
+
name = sib.get("rfilename", "")
|
|
58
|
+
ext = os.path.splitext(name)[1].lower()
|
|
59
|
+
display = f"{result.root}/{name}"
|
|
60
|
+
is_pickle = ext in PICKLE_EXTS or os.path.basename(name) in MODEL_PICKLE_NAMES
|
|
61
|
+
if is_pickle:
|
|
62
|
+
result.pickle_files_inspected += 1
|
|
63
|
+
size = sib.get("size")
|
|
64
|
+
if size is not None and size > max_bytes:
|
|
65
|
+
result.findings.append(Finding(
|
|
66
|
+
"HUB-PICKLE-TOO-LARGE", "high", display, None,
|
|
67
|
+
f"Pickle-family file of {size/1e6:.1f} MB exceeds the {max_bytes/1e6:.0f} MB "
|
|
68
|
+
"inspection cap and was NOT downloaded. Inspect it locally before trusting it.",
|
|
69
|
+
cve="CVE-2026-57845"))
|
|
70
|
+
continue
|
|
71
|
+
try:
|
|
72
|
+
data = fetch(base + urllib.request.quote(name))
|
|
73
|
+
except urllib.error.URLError as exc:
|
|
74
|
+
result.findings.append(Finding("HUB-DOWNLOAD-FAILED", "low", display, None,
|
|
75
|
+
f"Could not download for inspection: {exc}"))
|
|
76
|
+
continue
|
|
77
|
+
result.findings.extend(inspect_pickle_bytes(data, display))
|
|
78
|
+
elif ext == ".py":
|
|
79
|
+
result.files_scanned += 1
|
|
80
|
+
try:
|
|
81
|
+
source = fetch(base + urllib.request.quote(name)).decode("utf-8", "replace")
|
|
82
|
+
except urllib.error.URLError:
|
|
83
|
+
continue
|
|
84
|
+
result.findings.extend(scan_python_source(source, display))
|
|
85
|
+
if not siblings:
|
|
86
|
+
result.findings.append(Finding("HUB-EMPTY", "info", result.root, None,
|
|
87
|
+
"Hub API returned no files for this repository."))
|
|
88
|
+
return result
|