veritensor 1.9.1__tar.gz → 1.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {veritensor-1.9.1/src/veritensor.egg-info → veritensor-1.9.2}/PKG-INFO +3 -1
- {veritensor-1.9.1 → veritensor-1.9.2}/pyproject.toml +3 -2
- veritensor-1.9.2/src/veritensor/__init__.py +2 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/cli/main.py +32 -20
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/cache.py +27 -11
- veritensor-1.9.2/src/veritensor/core/networking.py +67 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/safe_zip.py +3 -4
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/streaming.py +19 -10
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/types.py +0 -7
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/content/pii.py +1 -1
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/data/dataset_engine.py +36 -19
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/data/excel_engine.py +1 -1
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/dependency_engine.py +13 -2
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/mcp_scanner.py +13 -5
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/notebook_engine.py +16 -8
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/rules.py +20 -14
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/enterprise_scanner.py +40 -21
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/huggingface.py +42 -33
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/compliance_report.py +14 -5
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/html_report.py +27 -9
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/sarif.py +30 -9
- {veritensor-1.9.1 → veritensor-1.9.2/src/veritensor.egg-info}/PKG-INFO +3 -1
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/requires.txt +4 -0
- veritensor-1.9.1/src/veritensor/__init__.py +0 -2
- veritensor-1.9.1/src/veritensor/core/networking.py +0 -64
- {veritensor-1.9.1 → veritensor-1.9.2}/LICENSE +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/MANIFEST.in +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/README.md +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/setup.cfg +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/__main__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/cli/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/config.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/entropy.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/file_utils.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/text_utils.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/container/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/container/archive_engine.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/content/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/content/injection.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/data/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/calculator.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/lfs.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/readers.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/keras_engine.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/mcp_permission_auditor.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/pickle_engine.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/signatures.yaml +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/__init__.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/chroma_guard.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/cosign.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/langchain_guard.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/llamaindex_guard.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/rag.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/unstructured_guard.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/excel_report.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/manifest.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/sbom.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/telemetry.py +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/SOURCES.txt +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/dependency_links.txt +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/entry_points.txt +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/top_level.txt +0 -0
- {veritensor-1.9.1 → veritensor-1.9.2}/veritensor.yaml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: veritensor
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.2
|
|
4
4
|
Summary: Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats.
|
|
5
5
|
Author-email: Arsenii Brazhnyk <veritensor@gmail.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -35,6 +35,8 @@ Requires-Dist: gitpython>=3.1.49
|
|
|
35
35
|
Requires-Dist: pyarrow>=14.0.0
|
|
36
36
|
Requires-Dist: pandas>=2.0.0
|
|
37
37
|
Requires-Dist: openpyxl>=3.1.0
|
|
38
|
+
Requires-Dist: defusedxml>=0.7.1
|
|
39
|
+
Requires-Dist: tomli>=2.0.0; python_version < "3.11"
|
|
38
40
|
Provides-Extra: test
|
|
39
41
|
Requires-Dist: pytest>=7.0.0; extra == "test"
|
|
40
42
|
Requires-Dist: pytest-mock>=3.10.0; extra == "test"
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "veritensor"
|
|
7
|
-
version = "1.9.
|
|
7
|
+
version = "1.9.2"
|
|
8
8
|
description = "Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -36,11 +36,12 @@ dependencies = [
|
|
|
36
36
|
"pillow>=12.2.0",
|
|
37
37
|
"lxml>=6.1.0",
|
|
38
38
|
"urllib3 >= 2.7.0",
|
|
39
|
-
# Fickling removed from here to avoid installing PyTorch (800MB+) for regular users
|
|
40
39
|
"gitpython>=3.1.49",
|
|
41
40
|
"pyarrow>=14.0.0",
|
|
42
41
|
"pandas>=2.0.0",
|
|
43
42
|
"openpyxl>=3.1.0",
|
|
43
|
+
"defusedxml>=0.7.1",
|
|
44
|
+
"tomli>=2.0.0; python_version < '3.11'",
|
|
44
45
|
]
|
|
45
46
|
|
|
46
47
|
|
|
@@ -11,6 +11,7 @@ import os
|
|
|
11
11
|
import datetime
|
|
12
12
|
import time
|
|
13
13
|
import requests
|
|
14
|
+
import copy
|
|
14
15
|
import fnmatch
|
|
15
16
|
import concurrent.futures
|
|
16
17
|
import multiprocessing
|
|
@@ -204,7 +205,8 @@ def is_suppressed(file_path: str, threat_msg: str, suppressions: List[dict]) ->
|
|
|
204
205
|
if not supp_path or not supp_type:
|
|
205
206
|
continue
|
|
206
207
|
file_name = Path(file_path).name
|
|
207
|
-
|
|
208
|
+
supp_name = Path(supp_path).name
|
|
209
|
+
if file_name == supp_name and supp_type in threat_msg:
|
|
208
210
|
return True
|
|
209
211
|
return False
|
|
210
212
|
|
|
@@ -239,6 +241,16 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
239
241
|
scan_res = ScanResult(file_path=file_path_str)
|
|
240
242
|
scan_res.repo_id = repo
|
|
241
243
|
|
|
244
|
+
# Check MCP config files early, before extension-based routing swallows .json files
|
|
245
|
+
if file_path and ext == ".json" and not is_s3:
|
|
246
|
+
try:
|
|
247
|
+
if is_mcp_config_file(file_path):
|
|
248
|
+
perm_result = audit_mcp_config(file_path)
|
|
249
|
+
for t in perm_result.to_threat_strings():
|
|
250
|
+
scan_res.add_threat(t)
|
|
251
|
+
except Exception as e:
|
|
252
|
+
logger.debug(f"MCP Audit failed for {file_name}: {e}")
|
|
253
|
+
|
|
242
254
|
# --- HYBRID ROUTING (Sending heavy files to the server) ---
|
|
243
255
|
# If the user has an Enterprise server connected
|
|
244
256
|
already_remote_scanned = False
|
|
@@ -360,12 +372,7 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
360
372
|
if mcp_result.mcp_tools_found:
|
|
361
373
|
for t in mcp_result.to_threat_strings():
|
|
362
374
|
scan_res.add_threat(t)
|
|
363
|
-
|
|
364
|
-
if is_mcp_config_file(file_path):
|
|
365
|
-
from veritensor.engines.static.mcp_permission_auditor import audit_mcp_config
|
|
366
|
-
perm_result = audit_mcp_config(file_path)
|
|
367
|
-
for t in perm_result.to_threat_strings():
|
|
368
|
-
scan_res.add_threat(t)
|
|
375
|
+
|
|
369
376
|
else:
|
|
370
377
|
# If the format is unknown to any engine at all.
|
|
371
378
|
# We add INFO, but DO NOT call add_threat(),
|
|
@@ -379,11 +386,6 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
379
386
|
except Exception as e:
|
|
380
387
|
scan_res.add_threat(f"CRITICAL: Engine Error: {str(e)}")
|
|
381
388
|
|
|
382
|
-
# --- MERGE REMOTE THREATS ---
|
|
383
|
-
for t in remote_threats:
|
|
384
|
-
if t not in scan_res.threats:
|
|
385
|
-
scan_res.add_threat(t)
|
|
386
|
-
|
|
387
389
|
# --- C. License Check ---
|
|
388
390
|
if not is_s3 and file_path:
|
|
389
391
|
reader = get_reader_for_file(file_path)
|
|
@@ -414,6 +416,11 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
|
|
|
414
416
|
|
|
415
417
|
return scan_res
|
|
416
418
|
|
|
419
|
+
def _worker_initializer():
|
|
420
|
+
"""Pre-load spaCy model once per worker process to avoid RAM spikes during scans."""
|
|
421
|
+
from veritensor.engines.content.pii import PIIScanner
|
|
422
|
+
PIIScanner.get_engine()
|
|
423
|
+
|
|
417
424
|
# --- SHARED SCAN LOGIC ---
|
|
418
425
|
def _run_scan_process(
|
|
419
426
|
paths: List[str], repo: Optional[str], jobs: Optional[int],
|
|
@@ -527,7 +534,10 @@ def _run_scan_process(
|
|
|
527
534
|
) as progress:
|
|
528
535
|
|
|
529
536
|
main_task = progress.add_task("Scanning...", total=len(tasks))
|
|
530
|
-
executor = concurrent.futures.ProcessPoolExecutor(
|
|
537
|
+
executor = concurrent.futures.ProcessPoolExecutor(
|
|
538
|
+
max_workers=jobs,
|
|
539
|
+
initializer=_worker_initializer
|
|
540
|
+
)
|
|
531
541
|
|
|
532
542
|
future_to_file = {
|
|
533
543
|
executor.submit(scan_worker, task_args): task_args[0]
|
|
@@ -583,14 +593,14 @@ def scan(
|
|
|
583
593
|
watch: bool = typer.Option(False, "--watch", "-w", help="Watch files for changes and scan automatically"),
|
|
584
594
|
|
|
585
595
|
):
|
|
586
|
-
"""
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
config =
|
|
596
|
+
"""Scans models, data, and code for security threats."""
|
|
597
|
+
# Work on a shallow copy to prevent mutating the singleton between runs
|
|
598
|
+
base_config = ConfigLoader.load()
|
|
599
|
+
config = copy.copy(base_config)
|
|
600
|
+
|
|
590
601
|
if verbose: logger.setLevel(logging.DEBUG)
|
|
591
602
|
is_machine_output = (json_output or sarif_output or sbom_output) and not output_file
|
|
592
603
|
|
|
593
|
-
# Passing the flags from the console to the global config object
|
|
594
604
|
config.report_url = report_to or config.report_url
|
|
595
605
|
config.api_key = api_key or config.api_key
|
|
596
606
|
|
|
@@ -637,7 +647,6 @@ def scan(
|
|
|
637
647
|
console.print(Panel.fit(f"🛡️ [bold cyan]Veritensor Security Scanner[/bold cyan] v{__version__}", border_style="cyan"))
|
|
638
648
|
|
|
639
649
|
if watch:
|
|
640
|
-
import time
|
|
641
650
|
console.print("[bold cyan]👀 Veritensor Watcher started. Monitoring files for changes... (Press Ctrl+C to stop)[/bold cyan]")
|
|
642
651
|
|
|
643
652
|
# Collecting the initial file modification dates
|
|
@@ -921,7 +930,10 @@ def _print_table(results: List[ScanResult]):
|
|
|
921
930
|
else:
|
|
922
931
|
unique_threats = list(dict.fromkeys(res.threats))
|
|
923
932
|
display_threats = "\n".join(unique_threats)
|
|
924
|
-
|
|
933
|
+
|
|
934
|
+
# Use Path.name for cross-platform compatibility
|
|
935
|
+
file_name = Path(res.file_path).name if res.file_path else "unknown"
|
|
936
|
+
table.add_row(file_name, f"[{status_style}]{res.status}[/{status_style}]", display_threats)
|
|
925
937
|
console.print(table)
|
|
926
938
|
|
|
927
939
|
def _perform_signing(image: str, status: str, config, timestamp: str, results: List[ScanResult]):
|
|
@@ -20,8 +20,11 @@ class HashCache:
|
|
|
20
20
|
if not self.conn:
|
|
21
21
|
return
|
|
22
22
|
try:
|
|
23
|
-
|
|
24
|
-
self.cursor.execute(
|
|
23
|
+
cutoff = time.time() - (days * 86400)
|
|
24
|
+
self.cursor.execute(
|
|
25
|
+
"DELETE FROM file_cache WHERE COALESCE(last_scanned, mtime) < ?",
|
|
26
|
+
(cutoff,)
|
|
27
|
+
)
|
|
25
28
|
self.conn.commit()
|
|
26
29
|
except Exception as e:
|
|
27
30
|
logger.debug(f"Cache cleanup failed: {e}")
|
|
@@ -35,11 +38,9 @@ class HashCache:
|
|
|
35
38
|
# check_same_thread=False allows using connection across threads.
|
|
36
39
|
# In our multiprocessing architecture, only the MAIN process interacts with this class.
|
|
37
40
|
self.conn = sqlite3.connect(str(CACHE_FILE), check_same_thread=False)
|
|
38
|
-
|
|
39
41
|
# Enable WAL mode for better performance and concurrency safety
|
|
40
42
|
self.conn.execute("PRAGMA journal_mode=WAL;")
|
|
41
43
|
self.conn.execute("PRAGMA synchronous=NORMAL;")
|
|
42
|
-
|
|
43
44
|
self.cursor = self.conn.cursor()
|
|
44
45
|
|
|
45
46
|
# Create table
|
|
@@ -48,14 +49,24 @@ class HashCache:
|
|
|
48
49
|
path TEXT PRIMARY KEY,
|
|
49
50
|
hash TEXT,
|
|
50
51
|
size INTEGER,
|
|
51
|
-
mtime REAL
|
|
52
|
+
mtime REAL,
|
|
53
|
+
last_scanned REAL
|
|
52
54
|
)
|
|
53
55
|
""")
|
|
56
|
+
|
|
57
|
+
# Handle migration for existing DBs
|
|
58
|
+
try:
|
|
59
|
+
self.cursor.execute("ALTER TABLE file_cache ADD COLUMN last_scanned REAL")
|
|
60
|
+
self.conn.commit()
|
|
61
|
+
except Exception:
|
|
62
|
+
pass # Column already exists
|
|
63
|
+
|
|
54
64
|
self.conn.commit()
|
|
55
65
|
except Exception as e:
|
|
56
66
|
logger.warning(f"Failed to initialize cache DB: {e}")
|
|
57
67
|
self.conn = None
|
|
58
68
|
|
|
69
|
+
|
|
59
70
|
def get(self, file_path: Path) -> Optional[str]:
|
|
60
71
|
"""Returns the hash if the file has not been changed."""
|
|
61
72
|
if not self.conn: return None
|
|
@@ -71,16 +82,21 @@ class HashCache:
|
|
|
71
82
|
row = self.cursor.fetchone()
|
|
72
83
|
|
|
73
84
|
if row:
|
|
74
|
-
cached_hash, cached_size, cached_mtime = row
|
|
75
|
-
# Compare size and mtime (float comparison usually safe for exact system mtime)
|
|
85
|
+
cached_hash, cached_size, cached_mtime = row[:3]
|
|
76
86
|
if cached_size == stats.st_size and cached_mtime == stats.st_mtime:
|
|
87
|
+
# Update last_scanned on cache hit
|
|
88
|
+
self.cursor.execute(
|
|
89
|
+
"UPDATE file_cache SET last_scanned = ? WHERE path = ?",
|
|
90
|
+
(time.time(), abs_path)
|
|
91
|
+
)
|
|
92
|
+
self.conn.commit()
|
|
77
93
|
return cached_hash
|
|
78
94
|
|
|
79
95
|
return None
|
|
80
96
|
except Exception:
|
|
81
|
-
# If file not found or permission error during stat
|
|
82
97
|
return None
|
|
83
98
|
|
|
99
|
+
|
|
84
100
|
def set(self, file_path: Path, file_hash: str):
|
|
85
101
|
"""Saves or updates the hash in the cache."""
|
|
86
102
|
if not self.conn: return
|
|
@@ -90,9 +106,9 @@ class HashCache:
|
|
|
90
106
|
stats = file_path.stat()
|
|
91
107
|
|
|
92
108
|
self.cursor.execute("""
|
|
93
|
-
INSERT OR REPLACE INTO file_cache (path, hash, size, mtime)
|
|
94
|
-
VALUES (?, ?, ?, ?)
|
|
95
|
-
""", (abs_path, file_hash, stats.st_size, stats.st_mtime))
|
|
109
|
+
INSERT OR REPLACE INTO file_cache (path, hash, size, mtime, last_scanned)
|
|
110
|
+
VALUES (?, ?, ?, ?, ?)
|
|
111
|
+
""", (abs_path, file_hash, stats.st_size, stats.st_mtime, time.time()))
|
|
96
112
|
|
|
97
113
|
self.conn.commit()
|
|
98
114
|
except Exception as e:
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Copyright 2026 Veritensor Security Apache 2.0
|
|
2
|
+
import socket
|
|
3
|
+
import ipaddress
|
|
4
|
+
import logging
|
|
5
|
+
import urllib3.util.connection as urllib3_cn
|
|
6
|
+
import requests
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
# Save the original connection function
|
|
11
|
+
_orig_create_connection = urllib3_cn.create_connection
|
|
12
|
+
|
|
13
|
+
def validate_ip_ssrf(ip_addr: str) -> bool:
|
|
14
|
+
"""Checks if an IP address is public and safe to connect to."""
|
|
15
|
+
try:
|
|
16
|
+
ip_obj = ipaddress.ip_address(ip_addr)
|
|
17
|
+
if (ip_obj.is_private or ip_obj.is_loopback or
|
|
18
|
+
ip_obj.is_link_local or ip_obj.is_multicast or
|
|
19
|
+
ip_obj.is_reserved or ip_obj.is_unspecified):
|
|
20
|
+
return False
|
|
21
|
+
return True
|
|
22
|
+
except ValueError:
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
def safe_create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, socket_options=None):
|
|
26
|
+
"""
|
|
27
|
+
Patched connection creator. Resolves DNS, validates against SSRF,
|
|
28
|
+
and forces the socket to connect to the safe IP, preventing TOCTOU DNS Rebinding.
|
|
29
|
+
TLS/SNI remains intact because urllib3 wraps the socket with the original hostname later.
|
|
30
|
+
"""
|
|
31
|
+
host, port = address
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
ip_list = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
|
|
35
|
+
except socket.gaierror:
|
|
36
|
+
raise ValueError(f"Could not resolve hostname: {host}")
|
|
37
|
+
|
|
38
|
+
safe_ip = None
|
|
39
|
+
for item in ip_list:
|
|
40
|
+
ip_addr = item[4][0]
|
|
41
|
+
if validate_ip_ssrf(ip_addr):
|
|
42
|
+
safe_ip = ip_addr
|
|
43
|
+
break
|
|
44
|
+
|
|
45
|
+
if not safe_ip:
|
|
46
|
+
raise ValueError(f"SSRF Protection: No safe public IP found for {host}. Access forbidden.")
|
|
47
|
+
|
|
48
|
+
# Connect to the resolved and validated IP
|
|
49
|
+
return _orig_create_connection((safe_ip, port), timeout, source_address, socket_options)
|
|
50
|
+
|
|
51
|
+
# Apply the patch globally for the CLI runtime
|
|
52
|
+
urllib3_cn.create_connection = safe_create_connection
|
|
53
|
+
|
|
54
|
+
def get_safe_session(url: str = None) -> requests.Session:
|
|
55
|
+
"""
|
|
56
|
+
Returns an SSRF-protected session.
|
|
57
|
+
The global urllib3 patch ensures all requests made by this session are safe.
|
|
58
|
+
"""
|
|
59
|
+
if url:
|
|
60
|
+
# Trigger a dry-run resolution check immediately if a URL is provided
|
|
61
|
+
from urllib.parse import urlparse
|
|
62
|
+
parsed = urlparse(url)
|
|
63
|
+
hostname = parsed.hostname
|
|
64
|
+
if hostname:
|
|
65
|
+
safe_create_connection((hostname, parsed.port or 80))
|
|
66
|
+
|
|
67
|
+
return requests.Session()
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
|
|
2
1
|
import zipfile
|
|
3
2
|
import logging
|
|
4
3
|
|
|
@@ -32,9 +31,9 @@ class SafeZipReader:
|
|
|
32
31
|
|
|
33
32
|
total_size += info.file_size
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
# Checking the total size
|
|
35
|
+
if total_size > SafeZipReader.MAX_UNZIPPED_SIZE:
|
|
36
|
+
raise ZipBombError(f"Zip Bomb detected! Total unzipped size {total_size} bytes exceeds limit.")
|
|
38
37
|
|
|
39
38
|
def read(self, zfile: zipfile.ZipFile, name: str) -> bytes:
|
|
40
39
|
info = zfile.getinfo(name)
|
|
@@ -29,7 +29,7 @@ class RemoteStream(io.IOBase):
|
|
|
29
29
|
def __init__(self, url: str, session: Optional[requests.Session] = None):
|
|
30
30
|
self._validate_url(url)
|
|
31
31
|
self.url = url
|
|
32
|
-
self.
|
|
32
|
+
self._session = session or get_safe_session(self.url)
|
|
33
33
|
self.pos = 0
|
|
34
34
|
self.size = self._fetch_size()
|
|
35
35
|
self._closed = False
|
|
@@ -60,10 +60,9 @@ class RemoteStream(io.IOBase):
|
|
|
60
60
|
def _fetch_size(self) -> int:
|
|
61
61
|
try:
|
|
62
62
|
headers = {"Range": "bytes=0-0"}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
resp = self._session.get(self.url, headers=headers, stream=True, timeout=10)
|
|
64
|
+
resp.raise_for_status()
|
|
65
|
+
|
|
67
66
|
content_range = resp.headers.get("Content-Range")
|
|
68
67
|
if content_range and "/" in content_range:
|
|
69
68
|
return int(content_range.split("/")[-1])
|
|
@@ -76,6 +75,7 @@ class RemoteStream(io.IOBase):
|
|
|
76
75
|
logger.error(f"Failed to fetch size for {self.url}: {e}")
|
|
77
76
|
raise
|
|
78
77
|
|
|
78
|
+
|
|
79
79
|
def read(self, size: int = -1) -> bytes:
|
|
80
80
|
if self._closed: raise ValueError("I/O operation on closed file.")
|
|
81
81
|
if self.pos >= self.size: return b""
|
|
@@ -89,9 +89,9 @@ class RemoteStream(io.IOBase):
|
|
|
89
89
|
|
|
90
90
|
headers = {"Range": f"bytes={self.pos}-{end}"}
|
|
91
91
|
try:
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
92
|
+
# Use safe session
|
|
93
|
+
resp = self._session.get(self.url, headers=headers, timeout=30)
|
|
94
|
+
resp.raise_for_status()
|
|
95
95
|
data = resp.content
|
|
96
96
|
self.pos += len(data)
|
|
97
97
|
return data
|
|
@@ -111,7 +111,12 @@ class RemoteStream(io.IOBase):
|
|
|
111
111
|
def tell(self) -> int: return self.pos
|
|
112
112
|
def seekable(self) -> bool: return True
|
|
113
113
|
def readable(self) -> bool: return True
|
|
114
|
-
def close(self):
|
|
114
|
+
def close(self):
|
|
115
|
+
# Properly close the session and call super().close()
|
|
116
|
+
if not self._closed:
|
|
117
|
+
self._closed = True
|
|
118
|
+
self._session.close()
|
|
119
|
+
super().close()
|
|
115
120
|
def __enter__(self): return self
|
|
116
121
|
def __exit__(self, exc_type, exc, tb): self.close()
|
|
117
122
|
|
|
@@ -189,7 +194,11 @@ class S3Stream(io.IOBase):
|
|
|
189
194
|
def tell(self) -> int: return self.pos
|
|
190
195
|
def seekable(self) -> bool: return True
|
|
191
196
|
def readable(self) -> bool: return True
|
|
192
|
-
def close(self):
|
|
197
|
+
def close(self):
|
|
198
|
+
# Call super().close() to properly update io.IOBase state
|
|
199
|
+
if not self._closed:
|
|
200
|
+
self._closed = True
|
|
201
|
+
super().close()
|
|
193
202
|
def __enter__(self): return self
|
|
194
203
|
def __exit__(self, exc_type, exc, tb): self.close()
|
|
195
204
|
|
|
@@ -19,7 +19,7 @@ class PIIScanner:
|
|
|
19
19
|
|
|
20
20
|
if PRESIDIO_AVAILABLE is None:
|
|
21
21
|
try:
|
|
22
|
-
#
|
|
22
|
+
# Lazy import inside the method to prevent DLL/OSError crashes on startup
|
|
23
23
|
from presidio_analyzer import AnalyzerEngine
|
|
24
24
|
from presidio_analyzer.nlp_engine import NlpEngineProvider
|
|
25
25
|
import spacy
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Copyright 2026 Veritensor Security Apache 2.0
|
|
2
2
|
# Dataset Scanner (Parquet, CSV, JSONL) for Data Poisoning, Malicious URLs, and Toxic Columns
|
|
3
|
-
|
|
3
|
+
import threading
|
|
4
4
|
import logging
|
|
5
5
|
import json
|
|
6
6
|
from pathlib import Path
|
|
@@ -31,23 +31,28 @@ FALLBACK_SUSPICIOUS = [
|
|
|
31
31
|
"regex:(?i)secret"
|
|
32
32
|
]
|
|
33
33
|
|
|
34
|
-
#
|
|
34
|
+
# Thread-safe lazy loading for GLiNER
|
|
35
35
|
_GLINER_UNAVAILABLE = object()
|
|
36
36
|
_gliner_model = None
|
|
37
|
+
_gliner_lock = threading.Lock()
|
|
37
38
|
|
|
38
39
|
def _get_gliner_model():
|
|
39
|
-
"""Lazy loads GLiNER to avoid 30-90s delay on every function call."""
|
|
40
40
|
global _gliner_model
|
|
41
|
-
if _gliner_model is None:
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
41
|
+
if _gliner_model is not None:
|
|
42
|
+
return _gliner_model
|
|
43
|
+
|
|
44
|
+
with _gliner_lock:
|
|
45
|
+
if _gliner_model is None: # Double-checked locking
|
|
46
|
+
try:
|
|
47
|
+
from gliner import GLiNER
|
|
48
|
+
logger.info("Loading GLiNER model for column analysis...")
|
|
49
|
+
_gliner_model = GLiNER.from_pretrained("urchade/gliner_multi-v2.1")
|
|
50
|
+
except ImportError:
|
|
51
|
+
_gliner_model = _GLINER_UNAVAILABLE
|
|
52
|
+
except Exception as e:
|
|
53
|
+
logger.warning(f"GLiNER failed to load: {e}. Falling back to regex.")
|
|
54
|
+
_gliner_model = _GLINER_UNAVAILABLE
|
|
55
|
+
|
|
51
56
|
return _gliner_model
|
|
52
57
|
|
|
53
58
|
def _check_toxic_columns(columns: List[str]) -> List[str]:
|
|
@@ -60,7 +65,8 @@ def _check_toxic_columns(columns: List[str]) -> List[str]:
|
|
|
60
65
|
|
|
61
66
|
toxic_keywords = [
|
|
62
67
|
"social_credit", "political_affiliation", "sexual_orientation",
|
|
63
|
-
"race", "religion", "trade_union", "biometric_categorization", "criminal_prediction"
|
|
68
|
+
"race", "religion", "trade_union", "biometric_categorization", "criminal_prediction",
|
|
69
|
+
"emotion_state", "sentiment_analysis_workplace", "facial_features", "cctv_face_id", "subliminal_metrics"
|
|
64
70
|
]
|
|
65
71
|
|
|
66
72
|
model = _get_gliner_model()
|
|
@@ -95,15 +101,19 @@ def scan_dataset(file_path: Path, full_scan: bool = False, bias_profile: Optiona
|
|
|
95
101
|
|
|
96
102
|
row_limit = None if full_scan else MAX_ROWS_DEFAULT
|
|
97
103
|
|
|
98
|
-
|
|
99
|
-
|
|
104
|
+
aggregator = None
|
|
105
|
+
if bias_profile:
|
|
106
|
+
try:
|
|
107
|
+
aggregator = BiasAggregator(bias_profile)
|
|
108
|
+
except ValueError as e:
|
|
109
|
+
logger.warning(f"Invalid bias profile for {file_path.name}: {e}")
|
|
100
110
|
|
|
101
111
|
try:
|
|
102
112
|
text_stream = None
|
|
103
113
|
|
|
104
114
|
if ext == ".parquet":
|
|
105
115
|
if not PYARROW_AVAILABLE:
|
|
106
|
-
return ["WARNING: pyarrow
|
|
116
|
+
return ["WARNING: pyarrow and pandas are required for local parquet scanning. Run 'pip install pyarrow pandas' or use the Enterprise Server."], None
|
|
107
117
|
|
|
108
118
|
try:
|
|
109
119
|
parquet_file = pq.ParquetFile(file_path)
|
|
@@ -268,9 +278,16 @@ def _extract_strings_from_json(data: Any) -> Generator[str, None, None]:
|
|
|
268
278
|
class BiasAggregator:
|
|
269
279
|
"""Aggregates counts for fairness metrics using O(1) memory. Safe for multiprocessing (no lambdas)."""
|
|
270
280
|
def __init__(self, profile: dict):
|
|
271
|
-
|
|
281
|
+
# Validate required fields to prevent AttributeError during processing
|
|
272
282
|
self.target_var = profile.get("target_variable")
|
|
273
|
-
|
|
283
|
+
if not self.target_var:
|
|
284
|
+
raise ValueError("BiasAggregator requires 'target_variable' in bias profile")
|
|
285
|
+
|
|
286
|
+
self.protected_attrs = profile.get("protected_attributes", [])
|
|
287
|
+
if not self.protected_attrs:
|
|
288
|
+
raise ValueError("BiasAggregator requires 'protected_attributes' in bias profile")
|
|
289
|
+
|
|
290
|
+
self.favorable_label = str(profile.get("favorable_label", "1"))
|
|
274
291
|
|
|
275
292
|
self.group_target_counts = {}
|
|
276
293
|
self.proxy_counts = {}
|
|
@@ -24,7 +24,7 @@ FORMULA_PREFIXES = ('=', '+', '-', '@')
|
|
|
24
24
|
def scan_excel(file_path: Path) -> List[str]:
|
|
25
25
|
threats = []
|
|
26
26
|
if not EXCEL_AVAILABLE:
|
|
27
|
-
return ["WARNING: openpyxl
|
|
27
|
+
return ["WARNING: openpyxl is required for local Excel scanning. Run 'pip install openpyxl' or use the Enterprise Server."]
|
|
28
28
|
|
|
29
29
|
try:
|
|
30
30
|
# read_only=True is faster and uses less memory
|
|
@@ -20,7 +20,11 @@ try:
|
|
|
20
20
|
# Python 3.11+ native TOML support
|
|
21
21
|
import tomllib
|
|
22
22
|
except ImportError:
|
|
23
|
-
|
|
23
|
+
try:
|
|
24
|
+
# Backport for Python 3.10
|
|
25
|
+
import tomli as tomllib
|
|
26
|
+
except ImportError:
|
|
27
|
+
tomllib = None
|
|
24
28
|
|
|
25
29
|
# --- Known Malware DB (MVP) ---
|
|
26
30
|
KNOWN_MALICIOUS = {
|
|
@@ -214,9 +218,15 @@ def _parse_requirements(path: Path) -> Dict[str, Optional[str]]:
|
|
|
214
218
|
if not line or line.startswith(("#", "-")):
|
|
215
219
|
continue
|
|
216
220
|
|
|
221
|
+
# Strip inline comments before any further parsing
|
|
222
|
+
comment_idx = line.find(' #')
|
|
223
|
+
if comment_idx != -1:
|
|
224
|
+
line = line[:comment_idx].strip()
|
|
225
|
+
if not line:
|
|
226
|
+
continue
|
|
227
|
+
|
|
217
228
|
# Processing direct links (git+, http) with #egg=
|
|
218
229
|
if "egg=" in line:
|
|
219
|
-
# Пример: git+https://github.com/user/repo.git#egg=evil_pkg
|
|
220
230
|
name_part = line.split("egg=")[-1].split("&")[0].strip()
|
|
221
231
|
deps[name_part] = None
|
|
222
232
|
continue
|
|
@@ -236,6 +246,7 @@ def _parse_requirements(path: Path) -> Dict[str, Optional[str]]:
|
|
|
236
246
|
return deps
|
|
237
247
|
|
|
238
248
|
|
|
249
|
+
|
|
239
250
|
def _parse_pyproject(path: Path) -> Dict[str, Optional[str]]:
|
|
240
251
|
deps = {}
|
|
241
252
|
try:
|
|
@@ -131,7 +131,6 @@ _SEVERITY_MAP = {
|
|
|
131
131
|
# Human-in-the-loop parameter names that indicate the tool has a confirmation gate
|
|
132
132
|
_HITL_PARAM_NAMES = frozenset({"confirm", "approved", "dry_run", "dryrun", "force"})
|
|
133
133
|
|
|
134
|
-
_STDLIB_ENTRYPOINTS = frozenset({"main", "__main__", "cli", "run", "start", "app"})
|
|
135
134
|
# ---------------------------------------------------------------------------
|
|
136
135
|
# Result types
|
|
137
136
|
# ---------------------------------------------------------------------------
|
|
@@ -216,13 +215,22 @@ class _MCPToolVisitor(ast.NodeVisitor):
|
|
|
216
215
|
|
|
217
216
|
|
|
218
217
|
def _is_mcp_tool(self, node: ast.FunctionDef) -> bool:
|
|
219
|
-
|
|
220
|
-
|
|
218
|
+
# Check decorator FIRST. If @tool is present, scan regardless of name.
|
|
219
|
+
has_mcp_decorator = False
|
|
221
220
|
for dec in node.decorator_list:
|
|
222
221
|
name = _decorator_base_name(dec)
|
|
223
222
|
if name and name.lower() in _MCP_TOOL_DECORATORS:
|
|
224
|
-
|
|
225
|
-
|
|
223
|
+
has_mcp_decorator = True
|
|
224
|
+
break
|
|
225
|
+
|
|
226
|
+
if not has_mcp_decorator:
|
|
227
|
+
return False
|
|
228
|
+
|
|
229
|
+
# Only skip dunder names (e.g., __init__) — they are never MCP tools
|
|
230
|
+
if node.name.startswith("__") and node.name.endswith("__"):
|
|
231
|
+
return False
|
|
232
|
+
|
|
233
|
+
return True
|
|
226
234
|
|
|
227
235
|
def _has_hitl_param(self, node: ast.FunctionDef) -> bool:
|
|
228
236
|
params = {arg.arg.lower() for arg in node.args.args}
|
|
@@ -10,6 +10,8 @@ import re
|
|
|
10
10
|
from veritensor.core.entropy import is_high_entropy
|
|
11
11
|
from veritensor.engines.static.rules import get_severity, SignatureLoader, is_match
|
|
12
12
|
from veritensor.engines.content.pii import PIIScanner
|
|
13
|
+
import tokenize
|
|
14
|
+
import io as _io
|
|
13
15
|
|
|
14
16
|
logger = logging.getLogger(__name__)
|
|
15
17
|
|
|
@@ -140,15 +142,21 @@ def _clean_magics(source: str) -> str:
|
|
|
140
142
|
return "\n".join(lines)
|
|
141
143
|
|
|
142
144
|
def _check_nesting_level(code: str, max_depth: int = 100) -> bool:
|
|
143
|
-
|
|
145
|
+
"""Check bracket depth, ignoring content inside string literals."""
|
|
144
146
|
depth = 0
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
if
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
147
|
+
try:
|
|
148
|
+
tokens = tokenize.generate_tokens(_io.StringIO(code).readline)
|
|
149
|
+
for tok_type, tok_string, _, _, _ in tokens:
|
|
150
|
+
if tok_type == tokenize.OP:
|
|
151
|
+
if tok_string in ('(', '[', '{'):
|
|
152
|
+
depth += 1
|
|
153
|
+
if depth > max_depth:
|
|
154
|
+
return False
|
|
155
|
+
elif tok_string in (')', ']', '}'):
|
|
156
|
+
depth = max(0, depth - 1)
|
|
157
|
+
except tokenize.TokenError:
|
|
158
|
+
# Incomplete code (common in notebooks) — allow it to proceed to AST
|
|
159
|
+
pass
|
|
152
160
|
return True
|
|
153
161
|
|
|
154
162
|
|