aisbom-cli 1.0.7__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/PKG-INFO +48 -5
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/README.md +44 -1
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/cli.py +120 -3
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/remote.py +43 -8
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/scanner.py +39 -10
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/pyproject.toml +4 -4
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/LICENSE +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/__init__.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/diff.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/linter.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/mock_generator.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/properties.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/safety.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/spdx_gen.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/telemetry.py +0 -0
- {aisbom_cli-1.0.7 → aisbom_cli-1.2.0}/aisbom/version_check.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: aisbom-cli
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: An AI Supply Chain security tool that that detects Pickle bombs and generates CycloneDX SBOMs for Machine Learning models.
|
|
5
5
|
License-File: LICENSE
|
|
6
6
|
Author: Ajoy L
|
|
@@ -11,14 +11,14 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.13
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
-
Requires-Dist: click (<8.
|
|
14
|
+
Requires-Dist: click (<8.5.0)
|
|
15
15
|
Requires-Dist: cyclonedx-python-lib (>=8.5,<12.0)
|
|
16
16
|
Requires-Dist: packaging (>=24,<27)
|
|
17
17
|
Requires-Dist: pip-requirements-parser (>=32.0.1,<33.0.0)
|
|
18
18
|
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
|
19
|
-
Requires-Dist: rich (>=13.7.1,<
|
|
19
|
+
Requires-Dist: rich (>=13.7.1,<16.0.0)
|
|
20
20
|
Requires-Dist: spdx-tools (>=0.8.3,<0.9.0)
|
|
21
|
-
Requires-Dist: typer[all] (>=0.12.5,<0.
|
|
21
|
+
Requires-Dist: typer[all] (>=0.12.5,<0.27.0)
|
|
22
22
|
Project-URL: Homepage, https://www.aisbom.io/
|
|
23
23
|
Project-URL: Repository, https://github.com/Lab700xOrg/aisbom
|
|
24
24
|
Description-Content-Type: text/markdown
|
|
@@ -123,6 +123,32 @@ aisbom scan hf://google-bert/bert-base-uncased
|
|
|
123
123
|
|
|
124
124
|
We use HTTP Range requests to inspect just the headers — scans complete in seconds and use zero disk. Verify SafeTensors compliance before you `git clone`.
|
|
125
125
|
|
|
126
|
+
### Authentication (private & gated Hugging Face models)
|
|
127
|
+
|
|
128
|
+
To scan a **private or gated** Hugging Face model, set a Hugging Face access token in the environment. AIsbom reads `HF_TOKEN` first, then `HUGGING_FACE_HUB_TOKEN` (the same precedence as `huggingface_hub`):
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
export HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxx
|
|
132
|
+
aisbom scan hf://your-org/private-model
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
The token is sent **only** to `huggingface.co` as a bearer credential on the model-metadata requests; it is dropped on the redirect to the presigned LFS CDN and is never attached to any other host. It is **never written to logs and never included in telemetry** — the only token-related field we emit is a `token_present` boolean (whether *a* token was set), never the value itself. See [Telemetry & Privacy](#telemetry--privacy).
|
|
136
|
+
|
|
137
|
+
In CI, supply the token from a secret and make sure the runner can reach Hugging Face:
|
|
138
|
+
|
|
139
|
+
```yaml
|
|
140
|
+
jobs:
|
|
141
|
+
scan:
|
|
142
|
+
runs-on: ubuntu-latest
|
|
143
|
+
steps:
|
|
144
|
+
- run: pip install aisbom-cli
|
|
145
|
+
- run: aisbom scan hf://your-org/private-model
|
|
146
|
+
env:
|
|
147
|
+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
> **Egress note:** hosted/firewalled CI runners must allow outbound HTTPS to `huggingface.co` **and** its LFS CDN (`cdn-lfs.huggingface.co` and the presigned object-storage hosts it redirects to). A blocked CDN hop surfaces as a network/timeout error, not an auth failure.
|
|
151
|
+
|
|
126
152
|
### Share a scan with your team
|
|
127
153
|
|
|
128
154
|
```bash
|
|
@@ -235,6 +261,23 @@ When the scan is clean, the comment collapses to a one-line ✅:
|
|
|
235
261
|
|
|
236
262
|
Re-runs update the same comment in place via a hidden `<!-- aisbom-action -->` marker — you'll never see stacked AIsbom comments on the same PR.
|
|
237
263
|
|
|
264
|
+
### Hosted dashboard (optional)
|
|
265
|
+
|
|
266
|
+
The PR comment shows you each scan; the hosted dashboard at [app.aisbom.io](https://app.aisbom.io) keeps the history. Set the optional `token` input and every scan's SBOM is posted to your inventory dashboard, where you can browse artifacts across repos and branches, track drift over time, and share an executive view with compliance stakeholders:
|
|
267
|
+
|
|
268
|
+
```yaml
|
|
269
|
+
- uses: Lab700xOrg/aisbom@v1
|
|
270
|
+
with:
|
|
271
|
+
directory: models/
|
|
272
|
+
token: ${{ secrets.AISBOM_TOKEN }}
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
Get a per-repo token at <https://app.aisbom.io/connect> (sign in with GitHub). Leave `token` unset and the Action stays purely local — nothing is sent to the dashboard.
|
|
276
|
+
|
|
277
|
+
#### Data flow & privacy
|
|
278
|
+
|
|
279
|
+
When (and only when) `token` is set, the Action POSTs the generated CycloneDX SBOM JSON to `https://app.aisbom.io/v1/scan-result`, along with the branch/tag name (`GITHUB_REF_NAME`) so the dashboard can attribute results to the right ref. That's the entire payload: the SBOM describes the *structure and findings* of your model files (names, hashes, licenses, risk levels) — never the weights or file contents, which don't leave the GitHub runner. Data is stored in the EU (Cloudflare R2/D1, EU jurisdiction). Every upload is announced in a loud log group in your CI output, so your logs always show when a network call happened and where the data went. To stop uploading, remove the `token` input — there is no background or implicit sending.
|
|
280
|
+
|
|
238
281
|
See [`action/README_ACTION.md`](action/README_ACTION.md) for the full inputs/outputs reference, permissions block, and troubleshooting.
|
|
239
282
|
|
|
240
283
|
---
|
|
@@ -304,7 +347,7 @@ If you explicitly use `--share`: the generated `sbom.json` document is uploaded
|
|
|
304
347
|
|
|
305
348
|
Per `aisbom diff`: a `cli_diff` event with `has_drift=true|false`.
|
|
306
349
|
|
|
307
|
-
On unhandled
|
|
350
|
+
On a scan fetch failure or unhandled exception: a `cli_error` event records the exception class name only (e.g. `JSONDecodeError`), a low-cardinality `http_status` bucket (e.g. `401`, `timeout`), and a `token_present` boolean (whether an `HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN` was set) — never the token value, the message, the traceback, a URL, or any file content.
|
|
308
351
|
|
|
309
352
|
Each event carries an anonymous `user_id` — a SHA-256 of your machine's MAC address plus an app salt, truncated to 16 hex chars. Stored in `~/.aisbom/config.json`. Lets us see returning users without identifying anyone.
|
|
310
353
|
|
|
@@ -98,6 +98,32 @@ aisbom scan hf://google-bert/bert-base-uncased
|
|
|
98
98
|
|
|
99
99
|
We use HTTP Range requests to inspect just the headers — scans complete in seconds and use zero disk. Verify SafeTensors compliance before you `git clone`.
|
|
100
100
|
|
|
101
|
+
### Authentication (private & gated Hugging Face models)
|
|
102
|
+
|
|
103
|
+
To scan a **private or gated** Hugging Face model, set a Hugging Face access token in the environment. AIsbom reads `HF_TOKEN` first, then `HUGGING_FACE_HUB_TOKEN` (the same precedence as `huggingface_hub`):
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
export HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxx
|
|
107
|
+
aisbom scan hf://your-org/private-model
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The token is sent **only** to `huggingface.co` as a bearer credential on the model-metadata requests; it is dropped on the redirect to the presigned LFS CDN and is never attached to any other host. It is **never written to logs and never included in telemetry** — the only token-related field we emit is a `token_present` boolean (whether *a* token was set), never the value itself. See [Telemetry & Privacy](#telemetry--privacy).
|
|
111
|
+
|
|
112
|
+
In CI, supply the token from a secret and make sure the runner can reach Hugging Face:
|
|
113
|
+
|
|
114
|
+
```yaml
|
|
115
|
+
jobs:
|
|
116
|
+
scan:
|
|
117
|
+
runs-on: ubuntu-latest
|
|
118
|
+
steps:
|
|
119
|
+
- run: pip install aisbom-cli
|
|
120
|
+
- run: aisbom scan hf://your-org/private-model
|
|
121
|
+
env:
|
|
122
|
+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
> **Egress note:** hosted/firewalled CI runners must allow outbound HTTPS to `huggingface.co` **and** its LFS CDN (`cdn-lfs.huggingface.co` and the presigned object-storage hosts it redirects to). A blocked CDN hop surfaces as a network/timeout error, not an auth failure.
|
|
126
|
+
|
|
101
127
|
### Share a scan with your team
|
|
102
128
|
|
|
103
129
|
```bash
|
|
@@ -210,6 +236,23 @@ When the scan is clean, the comment collapses to a one-line ✅:
|
|
|
210
236
|
|
|
211
237
|
Re-runs update the same comment in place via a hidden `<!-- aisbom-action -->` marker — you'll never see stacked AIsbom comments on the same PR.
|
|
212
238
|
|
|
239
|
+
### Hosted dashboard (optional)
|
|
240
|
+
|
|
241
|
+
The PR comment shows you each scan; the hosted dashboard at [app.aisbom.io](https://app.aisbom.io) keeps the history. Set the optional `token` input and every scan's SBOM is posted to your inventory dashboard, where you can browse artifacts across repos and branches, track drift over time, and share an executive view with compliance stakeholders:
|
|
242
|
+
|
|
243
|
+
```yaml
|
|
244
|
+
- uses: Lab700xOrg/aisbom@v1
|
|
245
|
+
with:
|
|
246
|
+
directory: models/
|
|
247
|
+
token: ${{ secrets.AISBOM_TOKEN }}
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Get a per-repo token at <https://app.aisbom.io/connect> (sign in with GitHub). Leave `token` unset and the Action stays purely local — nothing is sent to the dashboard.
|
|
251
|
+
|
|
252
|
+
#### Data flow & privacy
|
|
253
|
+
|
|
254
|
+
When (and only when) `token` is set, the Action POSTs the generated CycloneDX SBOM JSON to `https://app.aisbom.io/v1/scan-result`, along with the branch/tag name (`GITHUB_REF_NAME`) so the dashboard can attribute results to the right ref. That's the entire payload: the SBOM describes the *structure and findings* of your model files (names, hashes, licenses, risk levels) — never the weights or file contents, which don't leave the GitHub runner. Data is stored in the EU (Cloudflare R2/D1, EU jurisdiction). Every upload is announced in a loud log group in your CI output, so your logs always show when a network call happened and where the data went. To stop uploading, remove the `token` input — there is no background or implicit sending.
|
|
255
|
+
|
|
213
256
|
See [`action/README_ACTION.md`](action/README_ACTION.md) for the full inputs/outputs reference, permissions block, and troubleshooting.
|
|
214
257
|
|
|
215
258
|
---
|
|
@@ -279,7 +322,7 @@ If you explicitly use `--share`: the generated `sbom.json` document is uploaded
|
|
|
279
322
|
|
|
280
323
|
Per `aisbom diff`: a `cli_diff` event with `has_drift=true|false`.
|
|
281
324
|
|
|
282
|
-
On unhandled
|
|
325
|
+
On a scan fetch failure or unhandled exception: a `cli_error` event records the exception class name only (e.g. `JSONDecodeError`), a low-cardinality `http_status` bucket (e.g. `401`, `timeout`), and a `token_present` boolean (whether an `HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN` was set) — never the token value, the message, the traceback, a URL, or any file content.
|
|
283
326
|
|
|
284
327
|
Each event carries an anonymous `user_id` — a SHA-256 of your machine's MAC address plus an app salt, truncated to 16 hex chars. Stored in `~/.aisbom/config.json`. Lets us see returning users without identifying anyone.
|
|
285
328
|
|
|
@@ -14,6 +14,7 @@ from cyclonedx.output.json import JsonV1Dot5, JsonV1Dot6
|
|
|
14
14
|
from cyclonedx.factory.license import LicenseFactory
|
|
15
15
|
from .mock_generator import create_mock_malware_file, create_mock_restricted_file, create_mock_gguf, create_demo_diff_sboms, create_mock_broken_file
|
|
16
16
|
from pathlib import Path
|
|
17
|
+
from urllib.parse import urlparse
|
|
17
18
|
import importlib.metadata
|
|
18
19
|
from .scanner import DeepScanner
|
|
19
20
|
from .diff import SBOMDiff
|
|
@@ -31,6 +32,9 @@ app = typer.Typer()
|
|
|
31
32
|
# plain f-strings. Without it, things like "aisbom 1.0.3" or "v1.6" get
|
|
32
33
|
# partial cyan coloring that looks like markup bugs.
|
|
33
34
|
console = Console(highlight=False)
|
|
35
|
+
# Fetch-failure messages (#58) go to stderr so they don't pollute piped stdout
|
|
36
|
+
# (SBOM JSON, markdown) and survive `aisbom scan … > out.json`.
|
|
37
|
+
err_console = Console(stderr=True, highlight=False)
|
|
34
38
|
|
|
35
39
|
|
|
36
40
|
@app.callback(invoke_without_command=True)
|
|
@@ -128,6 +132,95 @@ def _classify_target(target: str) -> str:
|
|
|
128
132
|
return "local"
|
|
129
133
|
|
|
130
134
|
|
|
135
|
+
def _classify_http_status(exc: BaseException) -> str:
|
|
136
|
+
"""Bucket a fetch exception into a low-cardinality status label.
|
|
137
|
+
|
|
138
|
+
Returns the numeric HTTP status as a string for an HTTPError (e.g. "401",
|
|
139
|
+
"404"), "timeout" / "connection_error" for the corresponding network
|
|
140
|
+
failures, or "other" for anything else. Never includes a response body or
|
|
141
|
+
any URL — only the diagnostic bucket. Timeout is checked before
|
|
142
|
+
ConnectionError so a ConnectTimeout (a subclass of both) buckets as a
|
|
143
|
+
timeout.
|
|
144
|
+
"""
|
|
145
|
+
if isinstance(exc, requests.exceptions.HTTPError):
|
|
146
|
+
response = getattr(exc, "response", None)
|
|
147
|
+
status = getattr(response, "status_code", None)
|
|
148
|
+
if status is not None:
|
|
149
|
+
return str(status)
|
|
150
|
+
return "other"
|
|
151
|
+
if isinstance(exc, requests.exceptions.Timeout):
|
|
152
|
+
return "timeout"
|
|
153
|
+
if isinstance(exc, requests.exceptions.ConnectionError):
|
|
154
|
+
return "connection_error"
|
|
155
|
+
return "other"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _token_present() -> str:
|
|
159
|
+
"""Report whether an HF token env var is set, as "true"/"false".
|
|
160
|
+
|
|
161
|
+
Only presence is reported — the token value is never read into telemetry.
|
|
162
|
+
"""
|
|
163
|
+
has_token = bool(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"))
|
|
164
|
+
return "true" if has_token else "false"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _scan_error_payload(exc: BaseException, target: str) -> dict:
|
|
168
|
+
"""Build the `cli_error` telemetry payload for a scan-path failure.
|
|
169
|
+
|
|
170
|
+
Single source of truth for the event shape (parent #55) so every emit
|
|
171
|
+
site — the top-level `except` and the per-target fetch-failure path —
|
|
172
|
+
sends exactly the same low-cardinality keys and never a URL, repo id,
|
|
173
|
+
hostname, or response body.
|
|
174
|
+
"""
|
|
175
|
+
return {
|
|
176
|
+
"command": "scan",
|
|
177
|
+
"error_type": type(exc).__name__,
|
|
178
|
+
"http_status": _classify_http_status(exc),
|
|
179
|
+
"token_present": _token_present(),
|
|
180
|
+
"target_type": _classify_target(target),
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _format_fetch_error(exc: BaseException, url: str) -> str:
|
|
185
|
+
"""Status-aware, traceback-free message for a fetch failure.
|
|
186
|
+
|
|
187
|
+
Reactive (parent #55, #58): we make the request, then adapt the wording to
|
|
188
|
+
the *observed* status — the message never claims a cause it can't see. Auth
|
|
189
|
+
branches additionally split on whether a token was present, so a gated repo
|
|
190
|
+
with no token and a bad/insufficient token get different remediation.
|
|
191
|
+
"""
|
|
192
|
+
status = _classify_http_status(exc)
|
|
193
|
+
if url.startswith("hf://"):
|
|
194
|
+
# Resolve-time failure: the raw target is a repo id, not a byte URL.
|
|
195
|
+
host = "huggingface.co"
|
|
196
|
+
name = url[len("hf://"):] or url
|
|
197
|
+
else:
|
|
198
|
+
host = urlparse(url).hostname or "the remote host"
|
|
199
|
+
name = Path(urlparse(url).path).name or url
|
|
200
|
+
|
|
201
|
+
if status in ("401", "403"):
|
|
202
|
+
if _token_present() == "true":
|
|
203
|
+
return (
|
|
204
|
+
f"Authentication failed for {name} despite a token being set. "
|
|
205
|
+
"Verify HF_TOKEN is valid and has read access — you may need to "
|
|
206
|
+
"accept the model's license on huggingface.co."
|
|
207
|
+
)
|
|
208
|
+
return (
|
|
209
|
+
f"Authentication failed for {name}; the model appears to be private "
|
|
210
|
+
"or gated. Set HF_TOKEN with a token that has read access."
|
|
211
|
+
)
|
|
212
|
+
if status in ("timeout", "connection_error"):
|
|
213
|
+
return (
|
|
214
|
+
f"Network error reaching {host}. In CI, check that egress/firewall "
|
|
215
|
+
"allows HTTPS to huggingface.co and its LFS CDN."
|
|
216
|
+
)
|
|
217
|
+
if status == "404":
|
|
218
|
+
return f"{name} not found (HTTP 404); check the repo id / URL."
|
|
219
|
+
if status.isdigit():
|
|
220
|
+
return f"Failed to fetch {name} (HTTP {status})."
|
|
221
|
+
return f"Failed to fetch {name}."
|
|
222
|
+
|
|
223
|
+
|
|
131
224
|
def _summarize_model_format(artifacts: list[dict]) -> str:
|
|
132
225
|
"""Reduce per-artifact `framework` values to a single label.
|
|
133
226
|
|
|
@@ -332,9 +425,12 @@ def scan(
|
|
|
332
425
|
else:
|
|
333
426
|
results = scanner.scan()
|
|
334
427
|
except Exception as e:
|
|
428
|
+
# Safety net for unexpected scan crashes. Per-target *fetch* failures are
|
|
429
|
+
# caught inside the scanner now (recorded as structured errors, no
|
|
430
|
+
# traceback); anything reaching here is genuinely unexpected.
|
|
335
431
|
err_thread = telemetry.post_event(
|
|
336
432
|
"cli_error",
|
|
337
|
-
|
|
433
|
+
_scan_error_payload(e, target),
|
|
338
434
|
scan_id=scan_id,
|
|
339
435
|
)
|
|
340
436
|
_flush_telemetry_threads(telemetry_threads + [err_thread])
|
|
@@ -357,6 +453,24 @@ def scan(
|
|
|
357
453
|
if fail_on_risk and highest_risk >= 3:
|
|
358
454
|
exit_code = 2
|
|
359
455
|
|
|
456
|
+
# Reactive fetch-failure handling (#58). The scanner caught these per-target
|
|
457
|
+
# and kept going, so other targets still rendered above. For each one, print
|
|
458
|
+
# a clean status-aware message to stderr (no traceback) and emit a cli_error
|
|
459
|
+
# carrying the same diagnostic buckets as the top-level except — so a single
|
|
460
|
+
# failed scan emits both a cli_error (the failure) and the normal cli_scan
|
|
461
|
+
# (context) below. Intentional; see the slice notes.
|
|
462
|
+
fetch_failures = [e for e in results['errors'] if e.get('fetch_failure')]
|
|
463
|
+
for err in fetch_failures:
|
|
464
|
+
exc = err.get('exception')
|
|
465
|
+
err_console.print(
|
|
466
|
+
f"[bold red]✖[/bold red] {_format_fetch_error(exc, err.get('file', ''))}"
|
|
467
|
+
)
|
|
468
|
+
telemetry_threads.append(
|
|
469
|
+
telemetry.post_event(
|
|
470
|
+
"cli_error", _scan_error_payload(exc, target), scan_id=scan_id
|
|
471
|
+
)
|
|
472
|
+
)
|
|
473
|
+
|
|
360
474
|
# Telemetry: fire cli_scan plus any conditional follow-ups now that the
|
|
361
475
|
# scan completed and risk is known. Non-blocking; flushed before exit.
|
|
362
476
|
_risk_label = {3: "critical", 2: "medium", 1: "low", 0: "none"}.get(highest_risk, "unknown")
|
|
@@ -439,9 +553,12 @@ def scan(
|
|
|
439
553
|
if results['dependencies']:
|
|
440
554
|
console.print(f"\n📦 Found [bold]{len(results['dependencies'])}[/bold] Python libraries.")
|
|
441
555
|
|
|
442
|
-
|
|
556
|
+
# Parse errors only — fetch failures already printed their status-aware
|
|
557
|
+
# message to stderr above and don't fit the "Could not parse" framing.
|
|
558
|
+
parse_errors = [e for e in results['errors'] if not e.get('fetch_failure')]
|
|
559
|
+
if parse_errors:
|
|
443
560
|
console.print("\n[bold red]⚠️ Errors Encountered:[/bold red]")
|
|
444
|
-
for err in
|
|
561
|
+
for err in parse_errors:
|
|
445
562
|
console.print(f" - Could not parse [yellow]{err['file']}[/yellow]: {err['error']}")
|
|
446
563
|
|
|
447
564
|
# 3. Generate CycloneDX SBOM (Standard Compliance)
|
|
@@ -1,4 +1,34 @@
|
|
|
1
|
-
|
|
1
|
+
import os
|
|
2
|
+
from typing import List, Optional, Any, Dict
|
|
3
|
+
from urllib.parse import urlparse
|
|
4
|
+
|
|
5
|
+
# Per ADR-0001: bearer credentials are sent only to this exact host. HF's
|
|
6
|
+
# resolve endpoint 302-redirects byte fetches to a presigned LFS CDN / S3 host;
|
|
7
|
+
# requests' default cross-host auth-stripping drops the header on that hop, so
|
|
8
|
+
# the token is validated at huggingface.co and never leaks to the CDN.
|
|
9
|
+
_HF_HOST = "huggingface.co"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _hf_token() -> Optional[str]:
|
|
13
|
+
"""Read the HF access token from the environment only (ADR-0001).
|
|
14
|
+
|
|
15
|
+
Matches huggingface_hub precedence: HF_TOKEN, then HUGGING_FACE_HUB_TOKEN.
|
|
16
|
+
The cached ~/.cache/huggingface/token login file is deliberately not read.
|
|
17
|
+
"""
|
|
18
|
+
return os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _auth_headers(url: str) -> Dict[str, str]:
|
|
22
|
+
"""Per-request Authorization header, gated on an exact huggingface.co host match.
|
|
23
|
+
|
|
24
|
+
Returns an empty dict when there is no token or the host is anything other
|
|
25
|
+
than huggingface.co — the security guard that keeps the token off the CDN /
|
|
26
|
+
S3 / arbitrary-mirror hosts.
|
|
27
|
+
"""
|
|
28
|
+
token = _hf_token()
|
|
29
|
+
if token and urlparse(url).hostname == _HF_HOST:
|
|
30
|
+
return {"Authorization": f"Bearer {token}"}
|
|
31
|
+
return {}
|
|
2
32
|
|
|
3
33
|
|
|
4
34
|
class _RequestsStub:
|
|
@@ -29,7 +59,9 @@ class RemoteStream:
|
|
|
29
59
|
|
|
30
60
|
def _fetch_size(self) -> int:
|
|
31
61
|
# Use a range request to learn total size from Content-Range header
|
|
32
|
-
|
|
62
|
+
headers = {"Range": "bytes=0-0"}
|
|
63
|
+
headers.update(_auth_headers(self.url))
|
|
64
|
+
resp = self.session.get(self.url, headers=headers)
|
|
33
65
|
resp.raise_for_status()
|
|
34
66
|
content_range = resp.headers.get("Content-Range")
|
|
35
67
|
if content_range and "/" in content_range:
|
|
@@ -53,6 +85,7 @@ class RemoteStream:
|
|
|
53
85
|
end = min(self.pos + size - 1, self.size - 1)
|
|
54
86
|
|
|
55
87
|
headers = {"Range": f"bytes={self.pos}-{end}"}
|
|
88
|
+
headers.update(_auth_headers(self.url))
|
|
56
89
|
resp = self.session.get(self.url, headers=headers)
|
|
57
90
|
resp.raise_for_status()
|
|
58
91
|
data = resp.content
|
|
@@ -101,12 +134,14 @@ def resolve_huggingface_repo(repo_id: str) -> List[str]:
|
|
|
101
134
|
repo_id = repo_id[len("hf://") :]
|
|
102
135
|
|
|
103
136
|
api_url = f"https://huggingface.co/api/models/{repo_id}/tree/main"
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
137
|
+
# Let fetch failures propagate (no broad swallow): a 401/403 on a
|
|
138
|
+
# private/gated repo, a 404 typo, or a network error must surface to the
|
|
139
|
+
# scanner as a real error with a status-aware hint — returning [] here is
|
|
140
|
+
# what produced the silent "0 artifacts found" (#58). A successful 200 with
|
|
141
|
+
# no supported files still returns [] below, which is correct.
|
|
142
|
+
resp = requests.get(api_url, headers=_auth_headers(api_url))
|
|
143
|
+
resp.raise_for_status()
|
|
144
|
+
data = resp.json()
|
|
110
145
|
|
|
111
146
|
supported_exts = (".pt", ".pth", ".bin", ".safetensors", ".gguf")
|
|
112
147
|
urls = []
|
|
@@ -36,18 +36,31 @@ class DeepScanner:
|
|
|
36
36
|
def scan(self):
|
|
37
37
|
"""Orchestrates the scan of the directory."""
|
|
38
38
|
if self.is_remote:
|
|
39
|
-
|
|
39
|
+
# Resolving the repo is itself a network call; a 401/403/404 here
|
|
40
|
+
# must surface as a structured error (not a swallowed empty list or
|
|
41
|
+
# a raw traceback) so the CLI can render a status-aware hint.
|
|
42
|
+
try:
|
|
43
|
+
targets = self._resolve_remote_targets(self.root_path)
|
|
44
|
+
except Exception as e:
|
|
45
|
+
self._record_fetch_error(self.root_path, e)
|
|
46
|
+
targets = []
|
|
40
47
|
for url in targets:
|
|
41
48
|
ext = Path(url).suffix.lower()
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
# Per-target isolation: one gated/missing file in a multi-file
|
|
50
|
+
# repo records its error and continues, so the rest still scan.
|
|
51
|
+
try:
|
|
52
|
+
if ext in PYTORCH_EXTENSIONS:
|
|
53
|
+
with RemoteStream(url) as stream:
|
|
54
|
+
self.artifacts.append(self._inspect_pytorch(stream, Path(url).name, is_remote=True))
|
|
55
|
+
elif ext == SAFETENSORS_EXTENSION:
|
|
56
|
+
with RemoteStream(url) as stream:
|
|
57
|
+
self.artifacts.append(self._inspect_safetensors(stream, Path(url).name, is_remote=True))
|
|
58
|
+
elif ext == GGUF_EXTENSION:
|
|
59
|
+
with RemoteStream(url) as stream:
|
|
60
|
+
self.artifacts.append(self._inspect_gguf(stream, Path(url).name, is_remote=True))
|
|
61
|
+
except Exception as e:
|
|
62
|
+
self._record_fetch_error(url, e)
|
|
63
|
+
continue
|
|
51
64
|
else:
|
|
52
65
|
root = Path(self.root_path)
|
|
53
66
|
for full_path in root.rglob("*"):
|
|
@@ -72,6 +85,22 @@ class DeepScanner:
|
|
|
72
85
|
return [target]
|
|
73
86
|
return []
|
|
74
87
|
|
|
88
|
+
def _record_fetch_error(self, target: str, exc: Exception) -> None:
|
|
89
|
+
"""Record a remote fetch failure as a structured, non-fatal error.
|
|
90
|
+
|
|
91
|
+
Lands in results['errors'] so the CLI's `errors → exit 1` path fires.
|
|
92
|
+
Tagged `fetch_failure` (vs. a parse error) and carries the live
|
|
93
|
+
exception so the CLI can render a status-aware, traceback-free message
|
|
94
|
+
and emit the enriched cli_error telemetry. The exception object stays
|
|
95
|
+
in-process — errors are never serialized into the SBOM.
|
|
96
|
+
"""
|
|
97
|
+
self.errors.append({
|
|
98
|
+
"file": target,
|
|
99
|
+
"error": str(exc),
|
|
100
|
+
"fetch_failure": True,
|
|
101
|
+
"exception": exc,
|
|
102
|
+
})
|
|
103
|
+
|
|
75
104
|
def _calculate_hash(self, path: Path) -> str:
|
|
76
105
|
sha256_hash = hashlib.sha256()
|
|
77
106
|
try:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "aisbom-cli"
|
|
3
|
-
version = "1.0
|
|
3
|
+
version = "1.2.0"
|
|
4
4
|
description = "An AI Supply Chain security tool that that detects Pickle bombs and generates CycloneDX SBOMs for Machine Learning models."
|
|
5
5
|
authors = ["Ajoy L <lab700xdev@gmail.com>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -9,11 +9,11 @@ repository = "https://github.com/Lab700xOrg/aisbom"
|
|
|
9
9
|
urls = { "Homepage" = "https://www.aisbom.io/" }
|
|
10
10
|
[tool.poetry.dependencies]
|
|
11
11
|
python = "^3.11"
|
|
12
|
-
typer = {extras = ["all"], version = ">=0.12.5,<0.
|
|
13
|
-
rich = ">=13.7.1,<
|
|
12
|
+
typer = {extras = ["all"], version = ">=0.12.5,<0.27.0"}
|
|
13
|
+
rich = ">=13.7.1,<16.0.0"
|
|
14
14
|
cyclonedx-python-lib = ">=8.5,<12.0"
|
|
15
15
|
pip-requirements-parser = "^32.0.1"
|
|
16
|
-
click = "<8.
|
|
16
|
+
click = "<8.5.0"
|
|
17
17
|
spdx-tools = "^0.8.3"
|
|
18
18
|
packaging = ">=24,<27"
|
|
19
19
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|