veritensor 1.9.1__tar.gz → 1.9.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {veritensor-1.9.1/src/veritensor.egg-info → veritensor-1.9.3}/PKG-INFO +10 -33
- {veritensor-1.9.1 → veritensor-1.9.3}/README.md +2 -28
- {veritensor-1.9.1 → veritensor-1.9.3}/pyproject.toml +9 -7
- veritensor-1.9.3/src/veritensor/__init__.py +2 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/cli/main.py +100 -90
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/cache.py +28 -12
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/config.py +0 -5
- veritensor-1.9.3/src/veritensor/core/networking.py +84 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/safe_zip.py +7 -6
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/streaming.py +40 -17
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/types.py +0 -7
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/content/injection.py +1 -1
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/content/pii.py +9 -2
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/data/dataset_engine.py +46 -25
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/data/excel_engine.py +14 -11
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/calculator.py +12 -3
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/lfs.py +11 -9
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/readers.py +24 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/dependency_engine.py +23 -5
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/mcp_scanner.py +13 -5
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/notebook_engine.py +23 -11
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/pickle_engine.py +2 -4
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/rules.py +20 -14
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/chroma_guard.py +2 -1
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/enterprise_scanner.py +47 -30
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/huggingface.py +42 -33
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/rag.py +12 -3
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/unstructured_guard.py +2 -1
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/compliance_report.py +25 -12
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/html_report.py +32 -11
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/sarif.py +30 -9
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/sbom.py +5 -1
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/telemetry.py +14 -2
- {veritensor-1.9.1 → veritensor-1.9.3/src/veritensor.egg-info}/PKG-INFO +10 -33
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/SOURCES.txt +0 -1
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/requires.txt +9 -4
- veritensor-1.9.1/src/veritensor/__init__.py +0 -2
- veritensor-1.9.1/src/veritensor/core/networking.py +0 -64
- veritensor-1.9.1/src/veritensor/integrations/cosign.py +0 -105
- {veritensor-1.9.1 → veritensor-1.9.3}/LICENSE +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/MANIFEST.in +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/setup.cfg +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/__main__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/cli/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/entropy.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/file_utils.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/text_utils.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/container/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/container/archive_engine.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/content/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/data/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/keras_engine.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/mcp_permission_auditor.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/signatures.yaml +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/langchain_guard.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/llamaindex_guard.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/excel_report.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/manifest.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/dependency_links.txt +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/entry_points.txt +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/top_level.txt +0 -0
- {veritensor-1.9.1 → veritensor-1.9.3}/veritensor.yaml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: veritensor
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.3
|
|
4
4
|
Summary: Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats.
|
|
5
5
|
Author-email: Arsenii Brazhnyk <veritensor@gmail.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -28,13 +28,16 @@ Requires-Dist: pyyaml>=6.0
|
|
|
28
28
|
Requires-Dist: huggingface_hub>=0.19.0
|
|
29
29
|
Requires-Dist: cyclonedx-python-lib>=6.0.0
|
|
30
30
|
Requires-Dist: jinja2>=3.1.0
|
|
31
|
-
Requires-Dist: pillow>=12.
|
|
31
|
+
Requires-Dist: pillow>=12.3.0
|
|
32
32
|
Requires-Dist: lxml>=6.1.0
|
|
33
33
|
Requires-Dist: urllib3>=2.7.0
|
|
34
|
-
Requires-Dist: gitpython>=3.1.
|
|
34
|
+
Requires-Dist: gitpython>=3.1.54
|
|
35
35
|
Requires-Dist: pyarrow>=14.0.0
|
|
36
36
|
Requires-Dist: pandas>=2.0.0
|
|
37
37
|
Requires-Dist: openpyxl>=3.1.0
|
|
38
|
+
Requires-Dist: defusedxml>=0.7.1
|
|
39
|
+
Requires-Dist: tomli>=2.0.0; python_version < "3.11"
|
|
40
|
+
Requires-Dist: msgpack>=1.2.1
|
|
38
41
|
Provides-Extra: test
|
|
39
42
|
Requires-Dist: pytest>=7.0.0; extra == "test"
|
|
40
43
|
Requires-Dist: pytest-mock>=3.10.0; extra == "test"
|
|
@@ -46,14 +49,14 @@ Provides-Extra: pii
|
|
|
46
49
|
Requires-Dist: presidio-analyzer>=2.2.0; extra == "pii"
|
|
47
50
|
Requires-Dist: spacy>=3.0.0; extra == "pii"
|
|
48
51
|
Provides-Extra: rag
|
|
49
|
-
Requires-Dist: pypdf>=
|
|
52
|
+
Requires-Dist: pypdf>=6.14.2; extra == "rag"
|
|
50
53
|
Requires-Dist: python-docx>=1.1.0; extra == "rag"
|
|
51
54
|
Requires-Dist: python-pptx>=0.6.21; extra == "rag"
|
|
52
55
|
Requires-Dist: veritensor[pii]; extra == "rag"
|
|
53
56
|
Provides-Extra: aws
|
|
54
57
|
Requires-Dist: boto3>=1.28.0; extra == "aws"
|
|
55
58
|
Provides-Extra: all
|
|
56
|
-
Requires-Dist: veritensor[aws,
|
|
59
|
+
Requires-Dist: veritensor[aws,pii,rag]; extra == "all"
|
|
57
60
|
Provides-Extra: dev
|
|
58
61
|
Requires-Dist: veritensor[all,test]; extra == "dev"
|
|
59
62
|
Requires-Dist: black; extra == "dev"
|
|
@@ -395,32 +398,6 @@ veritensor scan ./models \
|
|
|
395
398
|
--sarif --output-file report.sarif \
|
|
396
399
|
--json --output-file results.json
|
|
397
400
|
```
|
|
398
|
-
---
|
|
399
|
-
|
|
400
|
-
## 🔐 Supply Chain Security (Container Signing)
|
|
401
|
-
|
|
402
|
-
Veritensor integrates with Sigstore Cosign to cryptographically sign your Docker images only if they pass the security scan.
|
|
403
|
-
|
|
404
|
-
### 1. Generate Keys
|
|
405
|
-
Generate a key pair for signing:
|
|
406
|
-
```bash
|
|
407
|
-
veritensor keygen
|
|
408
|
-
# Output: veritensor.key (Private) and veritensor.pub (Public)
|
|
409
|
-
```
|
|
410
|
-
### 2. Scan & Sign
|
|
411
|
-
Pass the --image flag and the path to your private key (via env var).
|
|
412
|
-
```bash
|
|
413
|
-
# Set path to your private key
|
|
414
|
-
export VERITENSOR_PRIVATE_KEY_PATH=veritensor.key
|
|
415
|
-
|
|
416
|
-
# If scan passes -> Sign the image
|
|
417
|
-
veritensor scan ./models/my_model.pkl --image my-org/my-app:v1.0.0
|
|
418
|
-
```
|
|
419
|
-
### 3. Verify (In Kubernetes / Production)
|
|
420
|
-
Before deploying, verify the signature to ensure the model was scanned:
|
|
421
|
-
```bash
|
|
422
|
-
cosign verify --key veritensor.pub my-org/my-app:v1.0.0
|
|
423
|
-
```
|
|
424
401
|
|
|
425
402
|
---
|
|
426
403
|
|
|
@@ -443,7 +420,7 @@ jobs:
|
|
|
443
420
|
steps:
|
|
444
421
|
- uses: actions/checkout@v4
|
|
445
422
|
- name: Veritensor Scan
|
|
446
|
-
uses: arsbr/Veritensor@v1.9.
|
|
423
|
+
uses: arsbr/Veritensor@v1.9.3
|
|
447
424
|
with:
|
|
448
425
|
path: '.'
|
|
449
426
|
jobs: '4'
|
|
@@ -454,7 +431,7 @@ Prevent committing malicious models to your repository. Add this to .pre-commit-
|
|
|
454
431
|
```yaml
|
|
455
432
|
repos:
|
|
456
433
|
- repo: https://github.com/arsbr/Veritensor
|
|
457
|
-
rev: v1.9.
|
|
434
|
+
rev: v1.9.3
|
|
458
435
|
hooks:
|
|
459
436
|
- id: veritensor-scan
|
|
460
437
|
```
|
|
@@ -333,32 +333,6 @@ veritensor scan ./models \
|
|
|
333
333
|
--sarif --output-file report.sarif \
|
|
334
334
|
--json --output-file results.json
|
|
335
335
|
```
|
|
336
|
-
---
|
|
337
|
-
|
|
338
|
-
## 🔐 Supply Chain Security (Container Signing)
|
|
339
|
-
|
|
340
|
-
Veritensor integrates with Sigstore Cosign to cryptographically sign your Docker images only if they pass the security scan.
|
|
341
|
-
|
|
342
|
-
### 1. Generate Keys
|
|
343
|
-
Generate a key pair for signing:
|
|
344
|
-
```bash
|
|
345
|
-
veritensor keygen
|
|
346
|
-
# Output: veritensor.key (Private) and veritensor.pub (Public)
|
|
347
|
-
```
|
|
348
|
-
### 2. Scan & Sign
|
|
349
|
-
Pass the --image flag and the path to your private key (via env var).
|
|
350
|
-
```bash
|
|
351
|
-
# Set path to your private key
|
|
352
|
-
export VERITENSOR_PRIVATE_KEY_PATH=veritensor.key
|
|
353
|
-
|
|
354
|
-
# If scan passes -> Sign the image
|
|
355
|
-
veritensor scan ./models/my_model.pkl --image my-org/my-app:v1.0.0
|
|
356
|
-
```
|
|
357
|
-
### 3. Verify (In Kubernetes / Production)
|
|
358
|
-
Before deploying, verify the signature to ensure the model was scanned:
|
|
359
|
-
```bash
|
|
360
|
-
cosign verify --key veritensor.pub my-org/my-app:v1.0.0
|
|
361
|
-
```
|
|
362
336
|
|
|
363
337
|
---
|
|
364
338
|
|
|
@@ -381,7 +355,7 @@ jobs:
|
|
|
381
355
|
steps:
|
|
382
356
|
- uses: actions/checkout@v4
|
|
383
357
|
- name: Veritensor Scan
|
|
384
|
-
uses: arsbr/Veritensor@v1.9.
|
|
358
|
+
uses: arsbr/Veritensor@v1.9.3
|
|
385
359
|
with:
|
|
386
360
|
path: '.'
|
|
387
361
|
jobs: '4'
|
|
@@ -392,7 +366,7 @@ Prevent committing malicious models to your repository. Add this to .pre-commit-
|
|
|
392
366
|
```yaml
|
|
393
367
|
repos:
|
|
394
368
|
- repo: https://github.com/arsbr/Veritensor
|
|
395
|
-
rev: v1.9.
|
|
369
|
+
rev: v1.9.3
|
|
396
370
|
hooks:
|
|
397
371
|
- id: veritensor-scan
|
|
398
372
|
```
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools>=
|
|
2
|
+
requires = ["setuptools>=78.1.1", "wheel>=0.46.2"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "veritensor"
|
|
7
|
-
version = "1.9.
|
|
7
|
+
version = "1.9.3"
|
|
8
8
|
description = "Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -33,14 +33,16 @@ dependencies = [
|
|
|
33
33
|
"huggingface_hub>=0.19.0",
|
|
34
34
|
"cyclonedx-python-lib>=6.0.0",
|
|
35
35
|
"jinja2>=3.1.0",
|
|
36
|
-
"pillow>=12.
|
|
36
|
+
"pillow>=12.3.0",
|
|
37
37
|
"lxml>=6.1.0",
|
|
38
38
|
"urllib3 >= 2.7.0",
|
|
39
|
-
|
|
40
|
-
"gitpython>=3.1.49",
|
|
39
|
+
"gitpython>=3.1.54",
|
|
41
40
|
"pyarrow>=14.0.0",
|
|
42
41
|
"pandas>=2.0.0",
|
|
43
42
|
"openpyxl>=3.1.0",
|
|
43
|
+
"defusedxml>=0.7.1",
|
|
44
|
+
"tomli>=2.0.0; python_version < '3.11'",
|
|
45
|
+
"msgpack>=1.2.1",
|
|
44
46
|
]
|
|
45
47
|
|
|
46
48
|
|
|
@@ -63,7 +65,7 @@ pii = [
|
|
|
63
65
|
|
|
64
66
|
# Optional dependencies for RAG document scanning (PDF, DOCX)
|
|
65
67
|
rag = [
|
|
66
|
-
"pypdf>=
|
|
68
|
+
"pypdf>=6.14.2",
|
|
67
69
|
"python-docx>=1.1.0",
|
|
68
70
|
"python-pptx>=0.6.21",
|
|
69
71
|
"veritensor[pii]"
|
|
@@ -75,7 +77,7 @@ aws = [
|
|
|
75
77
|
]
|
|
76
78
|
|
|
77
79
|
all = [
|
|
78
|
-
"veritensor[aws,
|
|
80
|
+
"veritensor[aws,rag,pii]"
|
|
79
81
|
]
|
|
80
82
|
|
|
81
83
|
dev = [
|
|
@@ -11,6 +11,7 @@ import os
|
|
|
11
11
|
import datetime
|
|
12
12
|
import time
|
|
13
13
|
import requests
|
|
14
|
+
import copy
|
|
14
15
|
import fnmatch
|
|
15
16
|
import concurrent.futures
|
|
16
17
|
import multiprocessing
|
|
@@ -60,7 +61,6 @@ except ImportError:
|
|
|
60
61
|
from veritensor.engines.static.rules import is_license_restricted
|
|
61
62
|
def is_match(repo, allowed): return False
|
|
62
63
|
|
|
63
|
-
from veritensor.integrations.cosign import sign_container, is_cosign_available, generate_key_pair
|
|
64
64
|
from veritensor.integrations.huggingface import HuggingFaceClient
|
|
65
65
|
|
|
66
66
|
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
@@ -109,7 +109,7 @@ HEAVY_EXTS = {
|
|
|
109
109
|
# --- AI Notebooks (AST, Secrets, Outputs) ---
|
|
110
110
|
".ipynb",
|
|
111
111
|
|
|
112
|
-
# --- Datasets (
|
|
112
|
+
# --- Datasets (sampled before upload to reduce payload size!) ---
|
|
113
113
|
".parquet", ".csv", ".tsv", ".jsonl", ".ndjson",
|
|
114
114
|
|
|
115
115
|
# --- Text & Markup (DeBERTa Semantic Scan, GLiNER) ---
|
|
@@ -131,6 +131,22 @@ HEAVY_EXTS = {
|
|
|
131
131
|
# Specific files that are sent to the server (name verification)
|
|
132
132
|
HEAVY_FILES = {"dockerfile"}
|
|
133
133
|
|
|
134
|
+
def _build_api_url(report_url: str, endpoint: str) -> str:
|
|
135
|
+
"""
|
|
136
|
+
Safely constructs API endpoint URL.
|
|
137
|
+
Expected input formats:
|
|
138
|
+
- http://server/api/v1
|
|
139
|
+
- http://server/api/v1/telemetry
|
|
140
|
+
"""
|
|
141
|
+
if not report_url:
|
|
142
|
+
return ""
|
|
143
|
+
base = report_url.rstrip("/")
|
|
144
|
+
for suffix in ("/api/v1/telemetry", "/telemetry", "/api/v1"):
|
|
145
|
+
if base.endswith(suffix):
|
|
146
|
+
base = base[: -len(suffix)]
|
|
147
|
+
break
|
|
148
|
+
|
|
149
|
+
return f"{base.rstrip('/')}/api/v1{endpoint}"
|
|
134
150
|
|
|
135
151
|
def load_ignore_patterns(ignore_file: str = ".veritensorignore") -> List[str]:
|
|
136
152
|
"""Loads glob patterns from .veritensorignore file."""
|
|
@@ -166,7 +182,7 @@ def check_remote_cache(report_url: str, api_key: str, hashes: List[str], version
|
|
|
166
182
|
if not report_url or not api_key or not hashes:
|
|
167
183
|
return {}
|
|
168
184
|
|
|
169
|
-
cache_url = report_url
|
|
185
|
+
cache_url = _build_api_url(report_url, "/cache/check")
|
|
170
186
|
headers = {"X-API-Key": api_key}
|
|
171
187
|
payload = {"hashes": hashes, "scanner_version": version} # Added a version to the payload
|
|
172
188
|
|
|
@@ -183,7 +199,7 @@ def fetch_server_policies(report_url: str, api_key: str) -> Tuple[Optional[dict]
|
|
|
183
199
|
if not report_url or not api_key:
|
|
184
200
|
return None,[]
|
|
185
201
|
|
|
186
|
-
policy_url = report_url
|
|
202
|
+
policy_url = _build_api_url(report_url, "/policies")
|
|
187
203
|
headers = {"X-API-Key": api_key}
|
|
188
204
|
|
|
189
205
|
try:
|
|
@@ -204,7 +220,8 @@ def is_suppressed(file_path: str, threat_msg: str, suppressions: List[dict]) ->
|
|
|
204
220
|
if not supp_path or not supp_type:
|
|
205
221
|
continue
|
|
206
222
|
file_name = Path(file_path).name
|
|
207
|
-
|
|
223
|
+
supp_name = Path(supp_path).name
|
|
224
|
+
if file_name == supp_name and supp_type in threat_msg:
|
|
208
225
|
return True
|
|
209
226
|
return False
|
|
210
227
|
|
|
@@ -239,6 +256,16 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
239
256
|
scan_res = ScanResult(file_path=file_path_str)
|
|
240
257
|
scan_res.repo_id = repo
|
|
241
258
|
|
|
259
|
+
# Check MCP config files early, before extension-based routing swallows .json files
|
|
260
|
+
if file_path and ext == ".json" and not is_s3:
|
|
261
|
+
try:
|
|
262
|
+
if is_mcp_config_file(file_path):
|
|
263
|
+
perm_result = audit_mcp_config(file_path)
|
|
264
|
+
for t in perm_result.to_threat_strings():
|
|
265
|
+
scan_res.add_threat(t)
|
|
266
|
+
except Exception as e:
|
|
267
|
+
logger.debug(f"MCP Audit failed for {file_name}: {e}")
|
|
268
|
+
|
|
242
269
|
# --- HYBRID ROUTING (Sending heavy files to the server) ---
|
|
243
270
|
# If the user has an Enterprise server connected
|
|
244
271
|
already_remote_scanned = False
|
|
@@ -252,12 +279,28 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
252
279
|
scanner = EnterpriseScanner(config.report_url, config.api_key)
|
|
253
280
|
remote_threats = scanner.scan_file_remotely(file_path, full_scan=full_scan_dataset)
|
|
254
281
|
|
|
255
|
-
|
|
256
|
-
|
|
282
|
+
# Detect network/timeout failures by inspecting returned warnings
|
|
283
|
+
_INFRA_FAILURE_PREFIXES = (
|
|
284
|
+
"WARNING: Failed to reach",
|
|
285
|
+
"WARNING: Enterprise scan timed out",
|
|
286
|
+
"WARNING: S3",
|
|
287
|
+
)
|
|
288
|
+
is_infra_failure = any(
|
|
289
|
+
t.startswith(_INFRA_FAILURE_PREFIXES) for t in remote_threats
|
|
290
|
+
) if remote_threats else False
|
|
257
291
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
292
|
+
if is_infra_failure:
|
|
293
|
+
# Infrastructure failure: do NOT set already_remote_scanned
|
|
294
|
+
# so local fallback can run for supported formats
|
|
295
|
+
logger.warning(f"Remote scan unavailable for {file_name}. Falling back to local scan.")
|
|
296
|
+
for t in remote_threats:
|
|
297
|
+
scan_res.threats.append(t) # Append without triggering immediate FAIL status
|
|
298
|
+
else:
|
|
299
|
+
if remote_threats:
|
|
300
|
+
for t in remote_threats:
|
|
301
|
+
scan_res.add_threat(t)
|
|
302
|
+
already_remote_scanned = True
|
|
303
|
+
|
|
261
304
|
except Exception as e:
|
|
262
305
|
scan_res.add_threat(f"WARNING: Remote scan failed, falling back to local: {e}")
|
|
263
306
|
|
|
@@ -318,8 +361,7 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
318
361
|
try:
|
|
319
362
|
|
|
320
363
|
safe_filename = quote(file_name, safe='')
|
|
321
|
-
prof_url = config.report_url
|
|
322
|
-
|
|
364
|
+
prof_url = _build_api_url(config.report_url, f"/fairness/profiles/{safe_filename}")
|
|
323
365
|
res = requests.get(prof_url, headers={"X-API-Key": config.api_key}, timeout=5)
|
|
324
366
|
if res.status_code == 200:
|
|
325
367
|
bias_profile = res.json().get("profile")
|
|
@@ -330,8 +372,9 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
330
372
|
# Pass the profile to the scanner
|
|
331
373
|
threats, bias_data = scan_dataset(file_path, full_scan=full_scan_dataset, bias_profile=bias_profile)
|
|
332
374
|
|
|
333
|
-
|
|
334
|
-
|
|
375
|
+
if not already_remote_scanned:
|
|
376
|
+
for t in threats:
|
|
377
|
+
scan_res.add_threat(t)
|
|
335
378
|
|
|
336
379
|
# Attach bias data to the result so it gets sent in telemetry
|
|
337
380
|
if bias_data:
|
|
@@ -360,12 +403,7 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
360
403
|
if mcp_result.mcp_tools_found:
|
|
361
404
|
for t in mcp_result.to_threat_strings():
|
|
362
405
|
scan_res.add_threat(t)
|
|
363
|
-
|
|
364
|
-
if is_mcp_config_file(file_path):
|
|
365
|
-
from veritensor.engines.static.mcp_permission_auditor import audit_mcp_config
|
|
366
|
-
perm_result = audit_mcp_config(file_path)
|
|
367
|
-
for t in perm_result.to_threat_strings():
|
|
368
|
-
scan_res.add_threat(t)
|
|
406
|
+
|
|
369
407
|
else:
|
|
370
408
|
# If the format is unknown to any engine at all.
|
|
371
409
|
# We add INFO, but DO NOT call add_threat(),
|
|
@@ -379,11 +417,6 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
379
417
|
except Exception as e:
|
|
380
418
|
scan_res.add_threat(f"CRITICAL: Engine Error: {str(e)}")
|
|
381
419
|
|
|
382
|
-
# --- MERGE REMOTE THREATS ---
|
|
383
|
-
for t in remote_threats:
|
|
384
|
-
if t not in scan_res.threats:
|
|
385
|
-
scan_res.add_threat(t)
|
|
386
|
-
|
|
387
420
|
# --- C. License Check ---
|
|
388
421
|
if not is_s3 and file_path:
|
|
389
422
|
reader = get_reader_for_file(file_path)
|
|
@@ -414,6 +447,11 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
414
447
|
|
|
415
448
|
return scan_res
|
|
416
449
|
|
|
450
|
+
def _worker_initializer():
|
|
451
|
+
"""Pre-load spaCy model once per worker process to avoid RAM spikes during scans."""
|
|
452
|
+
from veritensor.engines.content.pii import PIIScanner
|
|
453
|
+
PIIScanner.get_engine()
|
|
454
|
+
|
|
417
455
|
# --- SHARED SCAN LOGIC ---
|
|
418
456
|
def _run_scan_process(
|
|
419
457
|
paths: List[str], repo: Optional[str], jobs: Optional[int],
|
|
@@ -424,8 +462,8 @@ def _run_scan_process(
|
|
|
424
462
|
Core logic to collect files and run parallel scan.
|
|
425
463
|
Used by both 'scan' and 'manifest' commands.
|
|
426
464
|
"""
|
|
427
|
-
files_to_scan =[]
|
|
428
|
-
ignore_patterns = load_ignore_patterns()
|
|
465
|
+
files_to_scan: List[str] = []
|
|
466
|
+
ignore_patterns = load_ignore_patterns()
|
|
429
467
|
|
|
430
468
|
# 1. Collect Files from all paths
|
|
431
469
|
for path in paths:
|
|
@@ -436,11 +474,11 @@ def _run_scan_process(
|
|
|
436
474
|
local_path = Path(path)
|
|
437
475
|
if local_path.is_file():
|
|
438
476
|
if not is_ignored(local_path, ignore_patterns):
|
|
439
|
-
files_to_scan.append(local_path)
|
|
477
|
+
files_to_scan.append(str(local_path))
|
|
440
478
|
elif local_path.is_dir():
|
|
441
479
|
for p in local_path.rglob("*"):
|
|
442
480
|
if p.is_file() and not is_ignored(p, ignore_patterns):
|
|
443
|
-
files_to_scan.append(p)
|
|
481
|
+
files_to_scan.append(str(p))
|
|
444
482
|
else:
|
|
445
483
|
raise FileNotFoundError(f"Path {path} not found.")
|
|
446
484
|
|
|
@@ -460,21 +498,16 @@ def _run_scan_process(
|
|
|
460
498
|
tasks = []
|
|
461
499
|
|
|
462
500
|
# First, we quickly calculate the hashes locally (using SQLite so as not to read the files again)
|
|
463
|
-
local_hashes_map = {}
|
|
501
|
+
local_hashes_map = {}
|
|
502
|
+
|
|
464
503
|
for f in files_to_scan:
|
|
465
|
-
if
|
|
504
|
+
if f.startswith("s3://"):
|
|
466
505
|
continue
|
|
467
|
-
|
|
506
|
+
|
|
507
|
+
file_path_obj = Path(f)
|
|
508
|
+
cached_hash = hash_cache.get(file_path_obj)
|
|
468
509
|
if cached_hash:
|
|
469
|
-
local_hashes_map[
|
|
470
|
-
else:
|
|
471
|
-
# If there is no local cache, you will have to calculate
|
|
472
|
-
try:
|
|
473
|
-
h = calculate_sha256(f)
|
|
474
|
-
local_hashes_map[str(f)] = h
|
|
475
|
-
hash_cache.set(f, h)
|
|
476
|
-
except Exception:
|
|
477
|
-
pass
|
|
510
|
+
local_hashes_map[f] = cached_hash
|
|
478
511
|
|
|
479
512
|
# We ask the server about these hashes
|
|
480
513
|
remote_cache_results = {}
|
|
@@ -486,17 +519,17 @@ def _run_scan_process(
|
|
|
486
519
|
|
|
487
520
|
# Creating tasks
|
|
488
521
|
for f in files_to_scan:
|
|
489
|
-
is_s3 =
|
|
522
|
+
is_s3 = f.startswith("s3://")
|
|
490
523
|
if is_s3:
|
|
491
|
-
tasks.append((
|
|
524
|
+
tasks.append((f, config, repo, ignore_license, full_scan, True, None))
|
|
492
525
|
continue
|
|
493
526
|
|
|
494
|
-
file_hash = local_hashes_map.get(
|
|
527
|
+
file_hash = local_hashes_map.get(f)
|
|
495
528
|
|
|
496
529
|
# If the server knows this file, we DON'T add it to tasks
|
|
497
530
|
if file_hash and file_hash in remote_cache_results:
|
|
498
531
|
remote_data = remote_cache_results[file_hash]
|
|
499
|
-
res = ScanResult(file_path=
|
|
532
|
+
res = ScanResult(file_path=f, status=remote_data["status"], file_hash=file_hash)
|
|
500
533
|
res.threats = remote_data["threats"]
|
|
501
534
|
|
|
502
535
|
reader = get_reader_for_file(Path(f))
|
|
@@ -509,7 +542,7 @@ def _run_scan_process(
|
|
|
509
542
|
|
|
510
543
|
results.append(res)
|
|
511
544
|
else:
|
|
512
|
-
tasks.append((
|
|
545
|
+
tasks.append((f, config, repo, ignore_license, full_scan, False, file_hash))
|
|
513
546
|
|
|
514
547
|
# 3. Execute
|
|
515
548
|
executor = None
|
|
@@ -527,7 +560,10 @@ def _run_scan_process(
|
|
|
527
560
|
) as progress:
|
|
528
561
|
|
|
529
562
|
main_task = progress.add_task("Scanning...", total=len(tasks))
|
|
530
|
-
executor = concurrent.futures.ProcessPoolExecutor(
|
|
563
|
+
executor = concurrent.futures.ProcessPoolExecutor(
|
|
564
|
+
max_workers=jobs,
|
|
565
|
+
initializer=_worker_initializer
|
|
566
|
+
)
|
|
531
567
|
|
|
532
568
|
future_to_file = {
|
|
533
569
|
executor.submit(scan_worker, task_args): task_args[0]
|
|
@@ -563,7 +599,6 @@ def _run_scan_process(
|
|
|
563
599
|
def scan(
|
|
564
600
|
paths: List[str] = typer.Argument(..., help="Paths to files, directories, or S3 URLs"),
|
|
565
601
|
repo: Optional[str] = typer.Option(None, "--repo", "-r", help="Hugging Face Repo ID"),
|
|
566
|
-
image: Optional[str] = typer.Option(None, help="Docker image tag to sign"),
|
|
567
602
|
ignore_license: bool = typer.Option(False, "--ignore-license", help="Do not fail on license violations"),
|
|
568
603
|
ignore_malware: bool = typer.Option(False, "--ignore-malware", help="Do not fail on malware/policy violations"),
|
|
569
604
|
full_scan: bool = typer.Option(False, "--full-scan", help="Scan entire dataset (slow)."),
|
|
@@ -583,14 +618,14 @@ def scan(
|
|
|
583
618
|
watch: bool = typer.Option(False, "--watch", "-w", help="Watch files for changes and scan automatically"),
|
|
584
619
|
|
|
585
620
|
):
|
|
586
|
-
"""
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
config =
|
|
621
|
+
"""Scans models, data, and code for security threats."""
|
|
622
|
+
# Work on a shallow copy to prevent mutating the singleton between runs
|
|
623
|
+
base_config = ConfigLoader.load()
|
|
624
|
+
config = copy.copy(base_config)
|
|
625
|
+
|
|
590
626
|
if verbose: logger.setLevel(logging.DEBUG)
|
|
591
627
|
is_machine_output = (json_output or sarif_output or sbom_output) and not output_file
|
|
592
628
|
|
|
593
|
-
# Passing the flags from the console to the global config object
|
|
594
629
|
config.report_url = report_to or config.report_url
|
|
595
630
|
config.api_key = api_key or config.api_key
|
|
596
631
|
|
|
@@ -601,7 +636,7 @@ def scan(
|
|
|
601
636
|
try:
|
|
602
637
|
with open(policy_path, "r") as f:
|
|
603
638
|
yaml_content = f.read()
|
|
604
|
-
sync_url = config.report_url
|
|
639
|
+
sync_url = _build_api_url(config.report_url, "/policies/sync")
|
|
605
640
|
res = requests.post(sync_url, headers={"X-API-Key": config.api_key}, json={"yaml_content": yaml_content})
|
|
606
641
|
if res.status_code == 200:
|
|
607
642
|
console.print("[bold green]✅ Policy-as-Code successfully synced to Control Plane.[/bold green]")
|
|
@@ -637,7 +672,6 @@ def scan(
|
|
|
637
672
|
console.print(Panel.fit(f"🛡️ [bold cyan]Veritensor Security Scanner[/bold cyan] v{__version__}", border_style="cyan"))
|
|
638
673
|
|
|
639
674
|
if watch:
|
|
640
|
-
import time
|
|
641
675
|
console.print("[bold cyan]👀 Veritensor Watcher started. Monitoring files for changes... (Press Ctrl+C to stop)[/bold cyan]")
|
|
642
676
|
|
|
643
677
|
# Collecting the initial file modification dates
|
|
@@ -724,6 +758,8 @@ def scan(
|
|
|
724
758
|
|
|
725
759
|
if blocking_threats:
|
|
726
760
|
res.status = "FAIL"
|
|
761
|
+
elif any(t.startswith("WARNING: Failed to reach") or t.startswith("WARNING: Enterprise scan timed out") for t in real_threats):
|
|
762
|
+
res.status = "SCAN_ERROR"
|
|
727
763
|
else:
|
|
728
764
|
res.status = "PASS"
|
|
729
765
|
|
|
@@ -742,8 +778,9 @@ def scan(
|
|
|
742
778
|
if not res.threats: continue
|
|
743
779
|
|
|
744
780
|
artifact_name = Path(res.file_path).name
|
|
781
|
+
safe_artifact_name = quote(artifact_name, safe='')
|
|
745
782
|
fingerprint_param = f"?fingerprint={res.file_hash}" if res.file_hash else ""
|
|
746
|
-
baseline_url = config.report_url
|
|
783
|
+
baseline_url = _build_api_url(config.report_url, f"/scans/{safe_artifact_name}/baseline{fingerprint_param}")
|
|
747
784
|
|
|
748
785
|
try:
|
|
749
786
|
b_res = requests.get(baseline_url, headers={"X-API-Key": config.api_key}, timeout=60)
|
|
@@ -816,11 +853,11 @@ def scan(
|
|
|
816
853
|
|
|
817
854
|
machine_text = None
|
|
818
855
|
if sarif_output:
|
|
819
|
-
machine_text = generate_sarif_report(
|
|
856
|
+
machine_text = generate_sarif_report(filtered_results)
|
|
820
857
|
elif sbom_output:
|
|
821
|
-
machine_text = generate_sbom(
|
|
858
|
+
machine_text = generate_sbom(filtered_results)
|
|
822
859
|
elif json_output:
|
|
823
|
-
results_dicts = [r.__dict__ for r in
|
|
860
|
+
results_dicts = [r.__dict__ for r in filtered_results]
|
|
824
861
|
machine_text = json.dumps(results_dicts, indent=2)
|
|
825
862
|
|
|
826
863
|
if machine_text:
|
|
@@ -882,11 +919,6 @@ def scan(
|
|
|
882
919
|
else:
|
|
883
920
|
if not is_machine_output: console.print("\n[bold green]✅ Scan Passed.[/bold green]")
|
|
884
921
|
|
|
885
|
-
if image:
|
|
886
|
-
scan_timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
887
|
-
_perform_signing(image, sign_status, config, scan_timestamp, results)
|
|
888
|
-
|
|
889
|
-
|
|
890
922
|
@app.command()
|
|
891
923
|
def manifest(
|
|
892
924
|
paths: List[str] = typer.Argument(..., help="Paths to scan"),
|
|
@@ -921,34 +953,12 @@ def _print_table(results: List[ScanResult]):
|
|
|
921
953
|
else:
|
|
922
954
|
unique_threats = list(dict.fromkeys(res.threats))
|
|
923
955
|
display_threats = "\n".join(unique_threats)
|
|
924
|
-
|
|
956
|
+
|
|
957
|
+
# Use Path.name for cross-platform compatibility
|
|
958
|
+
file_name = Path(res.file_path).name if res.file_path else "unknown"
|
|
959
|
+
table.add_row(file_name, f"[{status_style}]{res.status}[/{status_style}]", display_threats)
|
|
925
960
|
console.print(table)
|
|
926
961
|
|
|
927
|
-
def _perform_signing(image: str, status: str, config, timestamp: str, results: List[ScanResult]):
|
|
928
|
-
console.print(f"\n🔐 [bold]Signing container:[/bold] {image}")
|
|
929
|
-
key_path = config.private_key_path or os.environ.get("VERITENSOR_PRIVATE_KEY_PATH")
|
|
930
|
-
if not key_path:
|
|
931
|
-
console.print("[red]Skipping signing: No private key found.[/red]")
|
|
932
|
-
return
|
|
933
|
-
annotations = {"scanned_by": "veritensor", "status": status, "scan_date": timestamp}
|
|
934
|
-
if results:
|
|
935
|
-
primary = results[0]
|
|
936
|
-
if primary.file_hash: annotations["ai.model.hash"] = primary.file_hash
|
|
937
|
-
if primary.detected_license: annotations["ai.model.license"] = primary.detected_license
|
|
938
|
-
if primary.repo_id: annotations["ai.model.source"] = primary.repo_id
|
|
939
|
-
success = sign_container(image, key_path, annotations=annotations)
|
|
940
|
-
if success: console.print(f"[green]✔ Signed with Smart Attestation.[/green]")
|
|
941
|
-
else: console.print(f"[bold red]Signing Failed.[/bold red]")
|
|
942
|
-
|
|
943
|
-
@app.command()
|
|
944
|
-
def keygen(output_prefix: str = "veritensor"):
|
|
945
|
-
console.print(f"[bold]Generating Cosign Key Pair ({output_prefix})...[/bold]")
|
|
946
|
-
if not is_cosign_available():
|
|
947
|
-
console.print("[bold red]Error:[/bold red] 'cosign' binary not found.")
|
|
948
|
-
raise typer.Exit(code=1)
|
|
949
|
-
if generate_key_pair(output_prefix): console.print(f"[green]✔ Keys generated.[/green]")
|
|
950
|
-
else: console.print("[red]Key generation failed.[/red]")
|
|
951
|
-
|
|
952
962
|
@app.command()
|
|
953
963
|
def update():
|
|
954
964
|
SIG_URL = "https://raw.githubusercontent.com/arsbr/Veritensor/main/src/veritensor/engines/static/signatures.yaml"
|