veritensor 1.9.2__tar.gz → 1.9.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {veritensor-1.9.2/src/veritensor.egg-info → veritensor-1.9.3}/PKG-INFO +8 -33
- {veritensor-1.9.2 → veritensor-1.9.3}/README.md +2 -28
- {veritensor-1.9.2 → veritensor-1.9.3}/pyproject.toml +7 -6
- veritensor-1.9.3/src/veritensor/__init__.py +2 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/cli/main.py +68 -70
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/cache.py +1 -1
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/config.py +0 -5
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/networking.py +23 -6
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/safe_zip.py +4 -2
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/streaming.py +27 -13
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/content/injection.py +1 -1
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/content/pii.py +8 -1
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/data/dataset_engine.py +10 -6
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/data/excel_engine.py +13 -10
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/hashing/calculator.py +12 -3
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/hashing/lfs.py +11 -9
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/hashing/readers.py +24 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/dependency_engine.py +14 -7
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/notebook_engine.py +7 -3
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/pickle_engine.py +2 -4
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/chroma_guard.py +2 -1
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/enterprise_scanner.py +7 -9
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/rag.py +12 -3
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/unstructured_guard.py +2 -1
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/reporting/compliance_report.py +11 -7
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/reporting/html_report.py +5 -2
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/reporting/sbom.py +5 -1
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/reporting/telemetry.py +14 -2
- {veritensor-1.9.2 → veritensor-1.9.3/src/veritensor.egg-info}/PKG-INFO +8 -33
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor.egg-info/SOURCES.txt +0 -1
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor.egg-info/requires.txt +5 -4
- veritensor-1.9.2/src/veritensor/__init__.py +0 -2
- veritensor-1.9.2/src/veritensor/integrations/cosign.py +0 -105
- {veritensor-1.9.2 → veritensor-1.9.3}/LICENSE +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/MANIFEST.in +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/setup.cfg +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/__main__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/cli/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/entropy.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/file_utils.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/text_utils.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/core/types.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/container/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/container/archive_engine.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/content/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/data/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/hashing/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/keras_engine.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/mcp_permission_auditor.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/mcp_scanner.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/rules.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/engines/static/signatures.yaml +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/__init__.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/huggingface.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/langchain_guard.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/integrations/llamaindex_guard.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/reporting/excel_report.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/reporting/manifest.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor/reporting/sarif.py +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor.egg-info/dependency_links.txt +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor.egg-info/entry_points.txt +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/src/veritensor.egg-info/top_level.txt +0 -0
- {veritensor-1.9.2 → veritensor-1.9.3}/veritensor.yaml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: veritensor
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.3
|
|
4
4
|
Summary: Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats.
|
|
5
5
|
Author-email: Arsenii Brazhnyk <veritensor@gmail.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -28,15 +28,16 @@ Requires-Dist: pyyaml>=6.0
|
|
|
28
28
|
Requires-Dist: huggingface_hub>=0.19.0
|
|
29
29
|
Requires-Dist: cyclonedx-python-lib>=6.0.0
|
|
30
30
|
Requires-Dist: jinja2>=3.1.0
|
|
31
|
-
Requires-Dist: pillow>=12.
|
|
31
|
+
Requires-Dist: pillow>=12.3.0
|
|
32
32
|
Requires-Dist: lxml>=6.1.0
|
|
33
33
|
Requires-Dist: urllib3>=2.7.0
|
|
34
|
-
Requires-Dist: gitpython>=3.1.
|
|
34
|
+
Requires-Dist: gitpython>=3.1.54
|
|
35
35
|
Requires-Dist: pyarrow>=14.0.0
|
|
36
36
|
Requires-Dist: pandas>=2.0.0
|
|
37
37
|
Requires-Dist: openpyxl>=3.1.0
|
|
38
38
|
Requires-Dist: defusedxml>=0.7.1
|
|
39
39
|
Requires-Dist: tomli>=2.0.0; python_version < "3.11"
|
|
40
|
+
Requires-Dist: msgpack>=1.2.1
|
|
40
41
|
Provides-Extra: test
|
|
41
42
|
Requires-Dist: pytest>=7.0.0; extra == "test"
|
|
42
43
|
Requires-Dist: pytest-mock>=3.10.0; extra == "test"
|
|
@@ -48,14 +49,14 @@ Provides-Extra: pii
|
|
|
48
49
|
Requires-Dist: presidio-analyzer>=2.2.0; extra == "pii"
|
|
49
50
|
Requires-Dist: spacy>=3.0.0; extra == "pii"
|
|
50
51
|
Provides-Extra: rag
|
|
51
|
-
Requires-Dist: pypdf>=
|
|
52
|
+
Requires-Dist: pypdf>=6.14.2; extra == "rag"
|
|
52
53
|
Requires-Dist: python-docx>=1.1.0; extra == "rag"
|
|
53
54
|
Requires-Dist: python-pptx>=0.6.21; extra == "rag"
|
|
54
55
|
Requires-Dist: veritensor[pii]; extra == "rag"
|
|
55
56
|
Provides-Extra: aws
|
|
56
57
|
Requires-Dist: boto3>=1.28.0; extra == "aws"
|
|
57
58
|
Provides-Extra: all
|
|
58
|
-
Requires-Dist: veritensor[aws,
|
|
59
|
+
Requires-Dist: veritensor[aws,pii,rag]; extra == "all"
|
|
59
60
|
Provides-Extra: dev
|
|
60
61
|
Requires-Dist: veritensor[all,test]; extra == "dev"
|
|
61
62
|
Requires-Dist: black; extra == "dev"
|
|
@@ -397,32 +398,6 @@ veritensor scan ./models \
|
|
|
397
398
|
--sarif --output-file report.sarif \
|
|
398
399
|
--json --output-file results.json
|
|
399
400
|
```
|
|
400
|
-
---
|
|
401
|
-
|
|
402
|
-
## 🔐 Supply Chain Security (Container Signing)
|
|
403
|
-
|
|
404
|
-
Veritensor integrates with Sigstore Cosign to cryptographically sign your Docker images only if they pass the security scan.
|
|
405
|
-
|
|
406
|
-
### 1. Generate Keys
|
|
407
|
-
Generate a key pair for signing:
|
|
408
|
-
```bash
|
|
409
|
-
veritensor keygen
|
|
410
|
-
# Output: veritensor.key (Private) and veritensor.pub (Public)
|
|
411
|
-
```
|
|
412
|
-
### 2. Scan & Sign
|
|
413
|
-
Pass the --image flag and the path to your private key (via env var).
|
|
414
|
-
```bash
|
|
415
|
-
# Set path to your private key
|
|
416
|
-
export VERITENSOR_PRIVATE_KEY_PATH=veritensor.key
|
|
417
|
-
|
|
418
|
-
# If scan passes -> Sign the image
|
|
419
|
-
veritensor scan ./models/my_model.pkl --image my-org/my-app:v1.0.0
|
|
420
|
-
```
|
|
421
|
-
### 3. Verify (In Kubernetes / Production)
|
|
422
|
-
Before deploying, verify the signature to ensure the model was scanned:
|
|
423
|
-
```bash
|
|
424
|
-
cosign verify --key veritensor.pub my-org/my-app:v1.0.0
|
|
425
|
-
```
|
|
426
401
|
|
|
427
402
|
---
|
|
428
403
|
|
|
@@ -445,7 +420,7 @@ jobs:
|
|
|
445
420
|
steps:
|
|
446
421
|
- uses: actions/checkout@v4
|
|
447
422
|
- name: Veritensor Scan
|
|
448
|
-
uses: arsbr/Veritensor@v1.9.
|
|
423
|
+
uses: arsbr/Veritensor@v1.9.3
|
|
449
424
|
with:
|
|
450
425
|
path: '.'
|
|
451
426
|
jobs: '4'
|
|
@@ -456,7 +431,7 @@ Prevent committing malicious models to your repository. Add this to .pre-commit-
|
|
|
456
431
|
```yaml
|
|
457
432
|
repos:
|
|
458
433
|
- repo: https://github.com/arsbr/Veritensor
|
|
459
|
-
rev: v1.9.
|
|
434
|
+
rev: v1.9.3
|
|
460
435
|
hooks:
|
|
461
436
|
- id: veritensor-scan
|
|
462
437
|
```
|
|
@@ -333,32 +333,6 @@ veritensor scan ./models \
|
|
|
333
333
|
--sarif --output-file report.sarif \
|
|
334
334
|
--json --output-file results.json
|
|
335
335
|
```
|
|
336
|
-
---
|
|
337
|
-
|
|
338
|
-
## 🔐 Supply Chain Security (Container Signing)
|
|
339
|
-
|
|
340
|
-
Veritensor integrates with Sigstore Cosign to cryptographically sign your Docker images only if they pass the security scan.
|
|
341
|
-
|
|
342
|
-
### 1. Generate Keys
|
|
343
|
-
Generate a key pair for signing:
|
|
344
|
-
```bash
|
|
345
|
-
veritensor keygen
|
|
346
|
-
# Output: veritensor.key (Private) and veritensor.pub (Public)
|
|
347
|
-
```
|
|
348
|
-
### 2. Scan & Sign
|
|
349
|
-
Pass the --image flag and the path to your private key (via env var).
|
|
350
|
-
```bash
|
|
351
|
-
# Set path to your private key
|
|
352
|
-
export VERITENSOR_PRIVATE_KEY_PATH=veritensor.key
|
|
353
|
-
|
|
354
|
-
# If scan passes -> Sign the image
|
|
355
|
-
veritensor scan ./models/my_model.pkl --image my-org/my-app:v1.0.0
|
|
356
|
-
```
|
|
357
|
-
### 3. Verify (In Kubernetes / Production)
|
|
358
|
-
Before deploying, verify the signature to ensure the model was scanned:
|
|
359
|
-
```bash
|
|
360
|
-
cosign verify --key veritensor.pub my-org/my-app:v1.0.0
|
|
361
|
-
```
|
|
362
336
|
|
|
363
337
|
---
|
|
364
338
|
|
|
@@ -381,7 +355,7 @@ jobs:
|
|
|
381
355
|
steps:
|
|
382
356
|
- uses: actions/checkout@v4
|
|
383
357
|
- name: Veritensor Scan
|
|
384
|
-
uses: arsbr/Veritensor@v1.9.
|
|
358
|
+
uses: arsbr/Veritensor@v1.9.3
|
|
385
359
|
with:
|
|
386
360
|
path: '.'
|
|
387
361
|
jobs: '4'
|
|
@@ -392,7 +366,7 @@ Prevent committing malicious models to your repository. Add this to .pre-commit-
|
|
|
392
366
|
```yaml
|
|
393
367
|
repos:
|
|
394
368
|
- repo: https://github.com/arsbr/Veritensor
|
|
395
|
-
rev: v1.9.
|
|
369
|
+
rev: v1.9.3
|
|
396
370
|
hooks:
|
|
397
371
|
- id: veritensor-scan
|
|
398
372
|
```
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools>=
|
|
2
|
+
requires = ["setuptools>=78.1.1", "wheel>=0.46.2"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "veritensor"
|
|
7
|
-
version = "1.9.
|
|
7
|
+
version = "1.9.3"
|
|
8
8
|
description = "Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -33,15 +33,16 @@ dependencies = [
|
|
|
33
33
|
"huggingface_hub>=0.19.0",
|
|
34
34
|
"cyclonedx-python-lib>=6.0.0",
|
|
35
35
|
"jinja2>=3.1.0",
|
|
36
|
-
"pillow>=12.
|
|
36
|
+
"pillow>=12.3.0",
|
|
37
37
|
"lxml>=6.1.0",
|
|
38
38
|
"urllib3 >= 2.7.0",
|
|
39
|
-
"gitpython>=3.1.
|
|
39
|
+
"gitpython>=3.1.54",
|
|
40
40
|
"pyarrow>=14.0.0",
|
|
41
41
|
"pandas>=2.0.0",
|
|
42
42
|
"openpyxl>=3.1.0",
|
|
43
43
|
"defusedxml>=0.7.1",
|
|
44
44
|
"tomli>=2.0.0; python_version < '3.11'",
|
|
45
|
+
"msgpack>=1.2.1",
|
|
45
46
|
]
|
|
46
47
|
|
|
47
48
|
|
|
@@ -64,7 +65,7 @@ pii = [
|
|
|
64
65
|
|
|
65
66
|
# Optional dependencies for RAG document scanning (PDF, DOCX)
|
|
66
67
|
rag = [
|
|
67
|
-
"pypdf>=
|
|
68
|
+
"pypdf>=6.14.2",
|
|
68
69
|
"python-docx>=1.1.0",
|
|
69
70
|
"python-pptx>=0.6.21",
|
|
70
71
|
"veritensor[pii]"
|
|
@@ -76,7 +77,7 @@ aws = [
|
|
|
76
77
|
]
|
|
77
78
|
|
|
78
79
|
all = [
|
|
79
|
-
"veritensor[aws,
|
|
80
|
+
"veritensor[aws,rag,pii]"
|
|
80
81
|
]
|
|
81
82
|
|
|
82
83
|
dev = [
|
|
@@ -61,7 +61,6 @@ except ImportError:
|
|
|
61
61
|
from veritensor.engines.static.rules import is_license_restricted
|
|
62
62
|
def is_match(repo, allowed): return False
|
|
63
63
|
|
|
64
|
-
from veritensor.integrations.cosign import sign_container, is_cosign_available, generate_key_pair
|
|
65
64
|
from veritensor.integrations.huggingface import HuggingFaceClient
|
|
66
65
|
|
|
67
66
|
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
@@ -110,7 +109,7 @@ HEAVY_EXTS = {
|
|
|
110
109
|
# --- AI Notebooks (AST, Secrets, Outputs) ---
|
|
111
110
|
".ipynb",
|
|
112
111
|
|
|
113
|
-
# --- Datasets (
|
|
112
|
+
# --- Datasets (sampled before upload to reduce payload size!) ---
|
|
114
113
|
".parquet", ".csv", ".tsv", ".jsonl", ".ndjson",
|
|
115
114
|
|
|
116
115
|
# --- Text & Markup (DeBERTa Semantic Scan, GLiNER) ---
|
|
@@ -132,6 +131,22 @@ HEAVY_EXTS = {
|
|
|
132
131
|
# Specific files that are sent to the server (name verification)
|
|
133
132
|
HEAVY_FILES = {"dockerfile"}
|
|
134
133
|
|
|
134
|
+
def _build_api_url(report_url: str, endpoint: str) -> str:
|
|
135
|
+
"""
|
|
136
|
+
Safely constructs API endpoint URL.
|
|
137
|
+
Expected input formats:
|
|
138
|
+
- http://server/api/v1
|
|
139
|
+
- http://server/api/v1/telemetry
|
|
140
|
+
"""
|
|
141
|
+
if not report_url:
|
|
142
|
+
return ""
|
|
143
|
+
base = report_url.rstrip("/")
|
|
144
|
+
for suffix in ("/api/v1/telemetry", "/telemetry", "/api/v1"):
|
|
145
|
+
if base.endswith(suffix):
|
|
146
|
+
base = base[: -len(suffix)]
|
|
147
|
+
break
|
|
148
|
+
|
|
149
|
+
return f"{base.rstrip('/')}/api/v1{endpoint}"
|
|
135
150
|
|
|
136
151
|
def load_ignore_patterns(ignore_file: str = ".veritensorignore") -> List[str]:
|
|
137
152
|
"""Loads glob patterns from .veritensorignore file."""
|
|
@@ -167,7 +182,7 @@ def check_remote_cache(report_url: str, api_key: str, hashes: List[str], version
|
|
|
167
182
|
if not report_url or not api_key or not hashes:
|
|
168
183
|
return {}
|
|
169
184
|
|
|
170
|
-
cache_url = report_url
|
|
185
|
+
cache_url = _build_api_url(report_url, "/cache/check")
|
|
171
186
|
headers = {"X-API-Key": api_key}
|
|
172
187
|
payload = {"hashes": hashes, "scanner_version": version} # Added a version to the payload
|
|
173
188
|
|
|
@@ -184,7 +199,7 @@ def fetch_server_policies(report_url: str, api_key: str) -> Tuple[Optional[dict]
|
|
|
184
199
|
if not report_url or not api_key:
|
|
185
200
|
return None,[]
|
|
186
201
|
|
|
187
|
-
policy_url = report_url
|
|
202
|
+
policy_url = _build_api_url(report_url, "/policies")
|
|
188
203
|
headers = {"X-API-Key": api_key}
|
|
189
204
|
|
|
190
205
|
try:
|
|
@@ -264,12 +279,28 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
264
279
|
scanner = EnterpriseScanner(config.report_url, config.api_key)
|
|
265
280
|
remote_threats = scanner.scan_file_remotely(file_path, full_scan=full_scan_dataset)
|
|
266
281
|
|
|
267
|
-
|
|
268
|
-
|
|
282
|
+
# Detect network/timeout failures by inspecting returned warnings
|
|
283
|
+
_INFRA_FAILURE_PREFIXES = (
|
|
284
|
+
"WARNING: Failed to reach",
|
|
285
|
+
"WARNING: Enterprise scan timed out",
|
|
286
|
+
"WARNING: S3",
|
|
287
|
+
)
|
|
288
|
+
is_infra_failure = any(
|
|
289
|
+
t.startswith(_INFRA_FAILURE_PREFIXES) for t in remote_threats
|
|
290
|
+
) if remote_threats else False
|
|
269
291
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
292
|
+
if is_infra_failure:
|
|
293
|
+
# Infrastructure failure: do NOT set already_remote_scanned
|
|
294
|
+
# so local fallback can run for supported formats
|
|
295
|
+
logger.warning(f"Remote scan unavailable for {file_name}. Falling back to local scan.")
|
|
296
|
+
for t in remote_threats:
|
|
297
|
+
scan_res.threats.append(t) # Append without triggering immediate FAIL status
|
|
298
|
+
else:
|
|
299
|
+
if remote_threats:
|
|
300
|
+
for t in remote_threats:
|
|
301
|
+
scan_res.add_threat(t)
|
|
302
|
+
already_remote_scanned = True
|
|
303
|
+
|
|
273
304
|
except Exception as e:
|
|
274
305
|
scan_res.add_threat(f"WARNING: Remote scan failed, falling back to local: {e}")
|
|
275
306
|
|
|
@@ -330,8 +361,7 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
330
361
|
try:
|
|
331
362
|
|
|
332
363
|
safe_filename = quote(file_name, safe='')
|
|
333
|
-
prof_url = config.report_url
|
|
334
|
-
|
|
364
|
+
prof_url = _build_api_url(config.report_url, f"/fairness/profiles/{safe_filename}")
|
|
335
365
|
res = requests.get(prof_url, headers={"X-API-Key": config.api_key}, timeout=5)
|
|
336
366
|
if res.status_code == 200:
|
|
337
367
|
bias_profile = res.json().get("profile")
|
|
@@ -342,8 +372,9 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
342
372
|
# Pass the profile to the scanner
|
|
343
373
|
threats, bias_data = scan_dataset(file_path, full_scan=full_scan_dataset, bias_profile=bias_profile)
|
|
344
374
|
|
|
345
|
-
|
|
346
|
-
|
|
375
|
+
if not already_remote_scanned:
|
|
376
|
+
for t in threats:
|
|
377
|
+
scan_res.add_threat(t)
|
|
347
378
|
|
|
348
379
|
# Attach bias data to the result so it gets sent in telemetry
|
|
349
380
|
if bias_data:
|
|
@@ -431,8 +462,8 @@ def _run_scan_process(
|
|
|
431
462
|
Core logic to collect files and run parallel scan.
|
|
432
463
|
Used by both 'scan' and 'manifest' commands.
|
|
433
464
|
"""
|
|
434
|
-
files_to_scan =[]
|
|
435
|
-
ignore_patterns = load_ignore_patterns()
|
|
465
|
+
files_to_scan: List[str] = []
|
|
466
|
+
ignore_patterns = load_ignore_patterns()
|
|
436
467
|
|
|
437
468
|
# 1. Collect Files from all paths
|
|
438
469
|
for path in paths:
|
|
@@ -443,11 +474,11 @@ def _run_scan_process(
|
|
|
443
474
|
local_path = Path(path)
|
|
444
475
|
if local_path.is_file():
|
|
445
476
|
if not is_ignored(local_path, ignore_patterns):
|
|
446
|
-
files_to_scan.append(local_path)
|
|
477
|
+
files_to_scan.append(str(local_path))
|
|
447
478
|
elif local_path.is_dir():
|
|
448
479
|
for p in local_path.rglob("*"):
|
|
449
480
|
if p.is_file() and not is_ignored(p, ignore_patterns):
|
|
450
|
-
files_to_scan.append(p)
|
|
481
|
+
files_to_scan.append(str(p))
|
|
451
482
|
else:
|
|
452
483
|
raise FileNotFoundError(f"Path {path} not found.")
|
|
453
484
|
|
|
@@ -467,21 +498,16 @@ def _run_scan_process(
|
|
|
467
498
|
tasks = []
|
|
468
499
|
|
|
469
500
|
# First, we quickly calculate the hashes locally (using SQLite so as not to read the files again)
|
|
470
|
-
local_hashes_map = {}
|
|
501
|
+
local_hashes_map = {}
|
|
502
|
+
|
|
471
503
|
for f in files_to_scan:
|
|
472
|
-
if
|
|
504
|
+
if f.startswith("s3://"):
|
|
473
505
|
continue
|
|
474
|
-
|
|
506
|
+
|
|
507
|
+
file_path_obj = Path(f)
|
|
508
|
+
cached_hash = hash_cache.get(file_path_obj)
|
|
475
509
|
if cached_hash:
|
|
476
|
-
local_hashes_map[
|
|
477
|
-
else:
|
|
478
|
-
# If there is no local cache, you will have to calculate
|
|
479
|
-
try:
|
|
480
|
-
h = calculate_sha256(f)
|
|
481
|
-
local_hashes_map[str(f)] = h
|
|
482
|
-
hash_cache.set(f, h)
|
|
483
|
-
except Exception:
|
|
484
|
-
pass
|
|
510
|
+
local_hashes_map[f] = cached_hash
|
|
485
511
|
|
|
486
512
|
# We ask the server about these hashes
|
|
487
513
|
remote_cache_results = {}
|
|
@@ -493,17 +519,17 @@ def _run_scan_process(
|
|
|
493
519
|
|
|
494
520
|
# Creating tasks
|
|
495
521
|
for f in files_to_scan:
|
|
496
|
-
is_s3 =
|
|
522
|
+
is_s3 = f.startswith("s3://")
|
|
497
523
|
if is_s3:
|
|
498
|
-
tasks.append((
|
|
524
|
+
tasks.append((f, config, repo, ignore_license, full_scan, True, None))
|
|
499
525
|
continue
|
|
500
526
|
|
|
501
|
-
file_hash = local_hashes_map.get(
|
|
527
|
+
file_hash = local_hashes_map.get(f)
|
|
502
528
|
|
|
503
529
|
# If the server knows this file, we DON'T add it to tasks
|
|
504
530
|
if file_hash and file_hash in remote_cache_results:
|
|
505
531
|
remote_data = remote_cache_results[file_hash]
|
|
506
|
-
res = ScanResult(file_path=
|
|
532
|
+
res = ScanResult(file_path=f, status=remote_data["status"], file_hash=file_hash)
|
|
507
533
|
res.threats = remote_data["threats"]
|
|
508
534
|
|
|
509
535
|
reader = get_reader_for_file(Path(f))
|
|
@@ -516,7 +542,7 @@ def _run_scan_process(
|
|
|
516
542
|
|
|
517
543
|
results.append(res)
|
|
518
544
|
else:
|
|
519
|
-
tasks.append((
|
|
545
|
+
tasks.append((f, config, repo, ignore_license, full_scan, False, file_hash))
|
|
520
546
|
|
|
521
547
|
# 3. Execute
|
|
522
548
|
executor = None
|
|
@@ -573,7 +599,6 @@ def _run_scan_process(
|
|
|
573
599
|
def scan(
|
|
574
600
|
paths: List[str] = typer.Argument(..., help="Paths to files, directories, or S3 URLs"),
|
|
575
601
|
repo: Optional[str] = typer.Option(None, "--repo", "-r", help="Hugging Face Repo ID"),
|
|
576
|
-
image: Optional[str] = typer.Option(None, help="Docker image tag to sign"),
|
|
577
602
|
ignore_license: bool = typer.Option(False, "--ignore-license", help="Do not fail on license violations"),
|
|
578
603
|
ignore_malware: bool = typer.Option(False, "--ignore-malware", help="Do not fail on malware/policy violations"),
|
|
579
604
|
full_scan: bool = typer.Option(False, "--full-scan", help="Scan entire dataset (slow)."),
|
|
@@ -611,7 +636,7 @@ def scan(
|
|
|
611
636
|
try:
|
|
612
637
|
with open(policy_path, "r") as f:
|
|
613
638
|
yaml_content = f.read()
|
|
614
|
-
sync_url = config.report_url
|
|
639
|
+
sync_url = _build_api_url(config.report_url, "/policies/sync")
|
|
615
640
|
res = requests.post(sync_url, headers={"X-API-Key": config.api_key}, json={"yaml_content": yaml_content})
|
|
616
641
|
if res.status_code == 200:
|
|
617
642
|
console.print("[bold green]✅ Policy-as-Code successfully synced to Control Plane.[/bold green]")
|
|
@@ -733,6 +758,8 @@ def scan(
|
|
|
733
758
|
|
|
734
759
|
if blocking_threats:
|
|
735
760
|
res.status = "FAIL"
|
|
761
|
+
elif any(t.startswith("WARNING: Failed to reach") or t.startswith("WARNING: Enterprise scan timed out") for t in real_threats):
|
|
762
|
+
res.status = "SCAN_ERROR"
|
|
736
763
|
else:
|
|
737
764
|
res.status = "PASS"
|
|
738
765
|
|
|
@@ -751,8 +778,9 @@ def scan(
|
|
|
751
778
|
if not res.threats: continue
|
|
752
779
|
|
|
753
780
|
artifact_name = Path(res.file_path).name
|
|
781
|
+
safe_artifact_name = quote(artifact_name, safe='')
|
|
754
782
|
fingerprint_param = f"?fingerprint={res.file_hash}" if res.file_hash else ""
|
|
755
|
-
baseline_url = config.report_url
|
|
783
|
+
baseline_url = _build_api_url(config.report_url, f"/scans/{safe_artifact_name}/baseline{fingerprint_param}")
|
|
756
784
|
|
|
757
785
|
try:
|
|
758
786
|
b_res = requests.get(baseline_url, headers={"X-API-Key": config.api_key}, timeout=60)
|
|
@@ -825,11 +853,11 @@ def scan(
|
|
|
825
853
|
|
|
826
854
|
machine_text = None
|
|
827
855
|
if sarif_output:
|
|
828
|
-
machine_text = generate_sarif_report(
|
|
856
|
+
machine_text = generate_sarif_report(filtered_results)
|
|
829
857
|
elif sbom_output:
|
|
830
|
-
machine_text = generate_sbom(
|
|
858
|
+
machine_text = generate_sbom(filtered_results)
|
|
831
859
|
elif json_output:
|
|
832
|
-
results_dicts = [r.__dict__ for r in
|
|
860
|
+
results_dicts = [r.__dict__ for r in filtered_results]
|
|
833
861
|
machine_text = json.dumps(results_dicts, indent=2)
|
|
834
862
|
|
|
835
863
|
if machine_text:
|
|
@@ -891,11 +919,6 @@ def scan(
|
|
|
891
919
|
else:
|
|
892
920
|
if not is_machine_output: console.print("\n[bold green]✅ Scan Passed.[/bold green]")
|
|
893
921
|
|
|
894
|
-
if image:
|
|
895
|
-
scan_timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
896
|
-
_perform_signing(image, sign_status, config, scan_timestamp, results)
|
|
897
|
-
|
|
898
|
-
|
|
899
922
|
@app.command()
|
|
900
923
|
def manifest(
|
|
901
924
|
paths: List[str] = typer.Argument(..., help="Paths to scan"),
|
|
@@ -936,31 +959,6 @@ def _print_table(results: List[ScanResult]):
|
|
|
936
959
|
table.add_row(file_name, f"[{status_style}]{res.status}[/{status_style}]", display_threats)
|
|
937
960
|
console.print(table)
|
|
938
961
|
|
|
939
|
-
def _perform_signing(image: str, status: str, config, timestamp: str, results: List[ScanResult]):
|
|
940
|
-
console.print(f"\n🔐 [bold]Signing container:[/bold] {image}")
|
|
941
|
-
key_path = config.private_key_path or os.environ.get("VERITENSOR_PRIVATE_KEY_PATH")
|
|
942
|
-
if not key_path:
|
|
943
|
-
console.print("[red]Skipping signing: No private key found.[/red]")
|
|
944
|
-
return
|
|
945
|
-
annotations = {"scanned_by": "veritensor", "status": status, "scan_date": timestamp}
|
|
946
|
-
if results:
|
|
947
|
-
primary = results[0]
|
|
948
|
-
if primary.file_hash: annotations["ai.model.hash"] = primary.file_hash
|
|
949
|
-
if primary.detected_license: annotations["ai.model.license"] = primary.detected_license
|
|
950
|
-
if primary.repo_id: annotations["ai.model.source"] = primary.repo_id
|
|
951
|
-
success = sign_container(image, key_path, annotations=annotations)
|
|
952
|
-
if success: console.print(f"[green]✔ Signed with Smart Attestation.[/green]")
|
|
953
|
-
else: console.print(f"[bold red]Signing Failed.[/bold red]")
|
|
954
|
-
|
|
955
|
-
@app.command()
|
|
956
|
-
def keygen(output_prefix: str = "veritensor"):
|
|
957
|
-
console.print(f"[bold]Generating Cosign Key Pair ({output_prefix})...[/bold]")
|
|
958
|
-
if not is_cosign_available():
|
|
959
|
-
console.print("[bold red]Error:[/bold red] 'cosign' binary not found.")
|
|
960
|
-
raise typer.Exit(code=1)
|
|
961
|
-
if generate_key_pair(output_prefix): console.print(f"[green]✔ Keys generated.[/green]")
|
|
962
|
-
else: console.print("[red]Key generation failed.[/red]")
|
|
963
|
-
|
|
964
962
|
@app.command()
|
|
965
963
|
def update():
|
|
966
964
|
SIG_URL = "https://raw.githubusercontent.com/arsbr/Veritensor/main/src/veritensor/engines/static/signatures.yaml"
|
|
@@ -17,7 +17,7 @@ class HashCache:
|
|
|
17
17
|
|
|
18
18
|
def _cleanup_old_entries(self, days: int = 30):
|
|
19
19
|
"""Remove cache entries not accessed in the last N days."""
|
|
20
|
-
if not self.conn:
|
|
20
|
+
if not self.conn or not hasattr(self, "cursor"):
|
|
21
21
|
return
|
|
22
22
|
try:
|
|
23
23
|
cutoff = time.time() - (days * 86400)
|
|
@@ -24,7 +24,6 @@ class VeritensorConfig:
|
|
|
24
24
|
ignored_rules: List[str] = field(default_factory=list)
|
|
25
25
|
fail_on_severity: str = "CRITICAL"
|
|
26
26
|
hf_token: Optional[str] = None
|
|
27
|
-
private_key_path: Optional[str] = None
|
|
28
27
|
output_format: str = "table"
|
|
29
28
|
fail_on_missing_license: bool = False
|
|
30
29
|
custom_restricted_licenses: List[str] = field(default_factory=list)
|
|
@@ -63,9 +62,6 @@ class ConfigLoader:
|
|
|
63
62
|
elif "HF_TOKEN" in os.environ:
|
|
64
63
|
config_data["hf_token"] = os.environ["HF_TOKEN"]
|
|
65
64
|
|
|
66
|
-
if "VERITENSOR_PRIVATE_KEY_PATH" in os.environ:
|
|
67
|
-
config_data["private_key_path"] = os.environ["VERITENSOR_PRIVATE_KEY_PATH"]
|
|
68
|
-
|
|
69
65
|
if "VERITENSOR_FAIL_ON" in os.environ:
|
|
70
66
|
config_data["fail_on_severity"] = os.environ["VERITENSOR_FAIL_ON"]
|
|
71
67
|
|
|
@@ -80,7 +76,6 @@ class ConfigLoader:
|
|
|
80
76
|
ignored_rules=config_data.get("ignored_rules", []),
|
|
81
77
|
fail_on_severity=config_data.get("fail_on_severity", "CRITICAL"),
|
|
82
78
|
hf_token=config_data.get("hf_token"),
|
|
83
|
-
private_key_path=config_data.get("private_key_path"),
|
|
84
79
|
output_format=config_data.get("output_format", "table"),
|
|
85
80
|
fail_on_missing_license=config_data.get("fail_on_missing_license", False),
|
|
86
81
|
custom_restricted_licenses=config_data.get("custom_restricted_licenses", []),
|
|
@@ -10,6 +10,27 @@ logger = logging.getLogger(__name__)
|
|
|
10
10
|
# Save the original connection function
|
|
11
11
|
_orig_create_connection = urllib3_cn.create_connection
|
|
12
12
|
|
|
13
|
+
def _validate_url_ssrf(url: str) -> None:
|
|
14
|
+
"""Validates that the URL resolves to a safe public IP. Does not connect."""
|
|
15
|
+
from urllib.parse import urlparse
|
|
16
|
+
parsed = urlparse(url)
|
|
17
|
+
hostname = parsed.hostname
|
|
18
|
+
if not hostname:
|
|
19
|
+
return
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
ip_list = socket.getaddrinfo(hostname, None, 0, socket.SOCK_STREAM)
|
|
23
|
+
except socket.gaierror:
|
|
24
|
+
raise ValueError(f"Cannot resolve hostname: {hostname}")
|
|
25
|
+
|
|
26
|
+
for item in ip_list:
|
|
27
|
+
ip_addr = item[4][0]
|
|
28
|
+
if validate_ip_ssrf(ip_addr):
|
|
29
|
+
return # At least one safe public IP found
|
|
30
|
+
|
|
31
|
+
# Raise exception if all resolved IPs are private/internal
|
|
32
|
+
raise ValueError(f"SSRF Protection: No safe public IP found for {hostname}. Access forbidden.")
|
|
33
|
+
|
|
13
34
|
def validate_ip_ssrf(ip_addr: str) -> bool:
|
|
14
35
|
"""Checks if an IP address is public and safe to connect to."""
|
|
15
36
|
try:
|
|
@@ -57,11 +78,7 @@ def get_safe_session(url: str = None) -> requests.Session:
|
|
|
57
78
|
The global urllib3 patch ensures all requests made by this session are safe.
|
|
58
79
|
"""
|
|
59
80
|
if url:
|
|
60
|
-
#
|
|
61
|
-
|
|
62
|
-
parsed = urlparse(url)
|
|
63
|
-
hostname = parsed.hostname
|
|
64
|
-
if hostname:
|
|
65
|
-
safe_create_connection((hostname, parsed.port or 80))
|
|
81
|
+
# Perform dry-run DNS validation without opening a TCP socket
|
|
82
|
+
_validate_url_ssrf(url)
|
|
66
83
|
|
|
67
84
|
return requests.Session()
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import zipfile
|
|
2
2
|
import logging
|
|
3
|
+
import os
|
|
3
4
|
|
|
4
5
|
logger = logging.getLogger(__name__)
|
|
5
6
|
|
|
@@ -20,8 +21,9 @@ class SafeZipReader:
|
|
|
20
21
|
total_size = 0
|
|
21
22
|
for info in zfile.infolist():
|
|
22
23
|
# Protection against endless file names
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
target = os.path.normpath(os.path.join("safe_root", info.filename.replace('\\', '/')))
|
|
25
|
+
if not target.startswith("safe_root" + os.sep) and target != "safe_root":
|
|
26
|
+
raise ZipBombError(f"Path traversal detected in ZIP entry: {info.filename}")
|
|
25
27
|
|
|
26
28
|
# Checking the compression ratio
|
|
27
29
|
if info.file_size > SafeZipReader.MIN_SIZE_FOR_RATIO and info.compress_size > 0:
|