veritensor 1.9.1__tar.gz → 1.9.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {veritensor-1.9.1/src/veritensor.egg-info → veritensor-1.9.2}/PKG-INFO +3 -1
  2. {veritensor-1.9.1 → veritensor-1.9.2}/pyproject.toml +3 -2
  3. veritensor-1.9.2/src/veritensor/__init__.py +2 -0
  4. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/cli/main.py +32 -20
  5. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/cache.py +27 -11
  6. veritensor-1.9.2/src/veritensor/core/networking.py +67 -0
  7. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/safe_zip.py +3 -4
  8. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/streaming.py +19 -10
  9. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/types.py +0 -7
  10. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/content/pii.py +1 -1
  11. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/data/dataset_engine.py +36 -19
  12. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/data/excel_engine.py +1 -1
  13. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/dependency_engine.py +13 -2
  14. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/mcp_scanner.py +13 -5
  15. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/notebook_engine.py +16 -8
  16. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/rules.py +20 -14
  17. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/enterprise_scanner.py +40 -21
  18. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/huggingface.py +42 -33
  19. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/compliance_report.py +14 -5
  20. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/html_report.py +27 -9
  21. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/sarif.py +30 -9
  22. {veritensor-1.9.1 → veritensor-1.9.2/src/veritensor.egg-info}/PKG-INFO +3 -1
  23. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/requires.txt +4 -0
  24. veritensor-1.9.1/src/veritensor/__init__.py +0 -2
  25. veritensor-1.9.1/src/veritensor/core/networking.py +0 -64
  26. {veritensor-1.9.1 → veritensor-1.9.2}/LICENSE +0 -0
  27. {veritensor-1.9.1 → veritensor-1.9.2}/MANIFEST.in +0 -0
  28. {veritensor-1.9.1 → veritensor-1.9.2}/README.md +0 -0
  29. {veritensor-1.9.1 → veritensor-1.9.2}/setup.cfg +0 -0
  30. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/__main__.py +0 -0
  31. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/cli/__init__.py +0 -0
  32. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/__init__.py +0 -0
  33. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/config.py +0 -0
  34. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/entropy.py +0 -0
  35. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/file_utils.py +0 -0
  36. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/core/text_utils.py +0 -0
  37. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/__init__.py +0 -0
  38. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/container/__init__.py +0 -0
  39. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/container/archive_engine.py +0 -0
  40. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/content/__init__.py +0 -0
  41. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/content/injection.py +0 -0
  42. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/data/__init__.py +0 -0
  43. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/__init__.py +0 -0
  44. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/calculator.py +0 -0
  45. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/lfs.py +0 -0
  46. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/hashing/readers.py +0 -0
  47. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/__init__.py +0 -0
  48. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/keras_engine.py +0 -0
  49. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/mcp_permission_auditor.py +0 -0
  50. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/pickle_engine.py +0 -0
  51. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/engines/static/signatures.yaml +0 -0
  52. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/__init__.py +0 -0
  53. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/chroma_guard.py +0 -0
  54. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/cosign.py +0 -0
  55. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/langchain_guard.py +0 -0
  56. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/llamaindex_guard.py +0 -0
  57. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/rag.py +0 -0
  58. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/integrations/unstructured_guard.py +0 -0
  59. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/excel_report.py +0 -0
  60. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/manifest.py +0 -0
  61. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/sbom.py +0 -0
  62. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor/reporting/telemetry.py +0 -0
  63. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/SOURCES.txt +0 -0
  64. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/dependency_links.txt +0 -0
  65. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/entry_points.txt +0 -0
  66. {veritensor-1.9.1 → veritensor-1.9.2}/src/veritensor.egg-info/top_level.txt +0 -0
  67. {veritensor-1.9.1 → veritensor-1.9.2}/veritensor.yaml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: veritensor
3
- Version: 1.9.1
3
+ Version: 1.9.2
4
4
  Summary: Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats.
5
5
  Author-email: Arsenii Brazhnyk <veritensor@gmail.com>
6
6
  License: Apache-2.0
@@ -35,6 +35,8 @@ Requires-Dist: gitpython>=3.1.49
35
35
  Requires-Dist: pyarrow>=14.0.0
36
36
  Requires-Dist: pandas>=2.0.0
37
37
  Requires-Dist: openpyxl>=3.1.0
38
+ Requires-Dist: defusedxml>=0.7.1
39
+ Requires-Dist: tomli>=2.0.0; python_version < "3.11"
38
40
  Provides-Extra: test
39
41
  Requires-Dist: pytest>=7.0.0; extra == "test"
40
42
  Requires-Dist: pytest-mock>=3.10.0; extra == "test"
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "veritensor"
7
- version = "1.9.1"
7
+ version = "1.9.2"
8
8
  description = "Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -36,11 +36,12 @@ dependencies = [
36
36
  "pillow>=12.2.0",
37
37
  "lxml>=6.1.0",
38
38
  "urllib3 >= 2.7.0",
39
- # Fickling removed from here to avoid installing PyTorch (800MB+) for regular users
40
39
  "gitpython>=3.1.49",
41
40
  "pyarrow>=14.0.0",
42
41
  "pandas>=2.0.0",
43
42
  "openpyxl>=3.1.0",
43
+ "defusedxml>=0.7.1",
44
+ "tomli>=2.0.0; python_version < '3.11'",
44
45
  ]
45
46
 
46
47
 
@@ -0,0 +1,2 @@
1
+ __version__ = "1.9.2"
2
+
@@ -11,6 +11,7 @@ import os
11
11
  import datetime
12
12
  import time
13
13
  import requests
14
+ import copy
14
15
  import fnmatch
15
16
  import concurrent.futures
16
17
  import multiprocessing
@@ -204,7 +205,8 @@ def is_suppressed(file_path: str, threat_msg: str, suppressions: List[dict]) ->
204
205
  if not supp_path or not supp_type:
205
206
  continue
206
207
  file_name = Path(file_path).name
207
- if file_name == supp_path and supp_type in threat_msg:
208
+ supp_name = Path(supp_path).name
209
+ if file_name == supp_name and supp_type in threat_msg:
208
210
  return True
209
211
  return False
210
212
 
@@ -239,6 +241,16 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
239
241
  scan_res = ScanResult(file_path=file_path_str)
240
242
  scan_res.repo_id = repo
241
243
 
244
+ # Check MCP config files early, before extension-based routing swallows .json files
245
+ if file_path and ext == ".json" and not is_s3:
246
+ try:
247
+ if is_mcp_config_file(file_path):
248
+ perm_result = audit_mcp_config(file_path)
249
+ for t in perm_result.to_threat_strings():
250
+ scan_res.add_threat(t)
251
+ except Exception as e:
252
+ logger.debug(f"MCP Audit failed for {file_name}: {e}")
253
+
242
254
  # --- HYBRID ROUTING (Sending heavy files to the server) ---
243
255
  # If the user has an Enterprise server connected
244
256
  already_remote_scanned = False
@@ -360,12 +372,7 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
360
372
  if mcp_result.mcp_tools_found:
361
373
  for t in mcp_result.to_threat_strings():
362
374
  scan_res.add_threat(t)
363
- elif ext == ".json" and file_path:
364
- if is_mcp_config_file(file_path):
365
- from veritensor.engines.static.mcp_permission_auditor import audit_mcp_config
366
- perm_result = audit_mcp_config(file_path)
367
- for t in perm_result.to_threat_strings():
368
- scan_res.add_threat(t)
375
+
369
376
  else:
370
377
  # If the format is unknown to any engine at all.
371
378
  # We add INFO, but DO NOT call add_threat(),
@@ -379,11 +386,6 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
379
386
  except Exception as e:
380
387
  scan_res.add_threat(f"CRITICAL: Engine Error: {str(e)}")
381
388
 
382
- # --- MERGE REMOTE THREATS ---
383
- for t in remote_threats:
384
- if t not in scan_res.threats:
385
- scan_res.add_threat(t)
386
-
387
389
  # --- C. License Check ---
388
390
  if not is_s3 and file_path:
389
391
  reader = get_reader_for_file(file_path)
@@ -414,6 +416,11 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
414
416
 
415
417
  return scan_res
416
418
 
419
+ def _worker_initializer():
420
+ """Pre-load spaCy model once per worker process to avoid RAM spikes during scans."""
421
+ from veritensor.engines.content.pii import PIIScanner
422
+ PIIScanner.get_engine()
423
+
417
424
  # --- SHARED SCAN LOGIC ---
418
425
  def _run_scan_process(
419
426
  paths: List[str], repo: Optional[str], jobs: Optional[int],
@@ -527,7 +534,10 @@ def _run_scan_process(
527
534
  ) as progress:
528
535
 
529
536
  main_task = progress.add_task("Scanning...", total=len(tasks))
530
- executor = concurrent.futures.ProcessPoolExecutor(max_workers=jobs)
537
+ executor = concurrent.futures.ProcessPoolExecutor(
538
+ max_workers=jobs,
539
+ initializer=_worker_initializer
540
+ )
531
541
 
532
542
  future_to_file = {
533
543
  executor.submit(scan_worker, task_args): task_args[0]
@@ -583,14 +593,14 @@ def scan(
583
593
  watch: bool = typer.Option(False, "--watch", "-w", help="Watch files for changes and scan automatically"),
584
594
 
585
595
  ):
586
- """
587
- Scans models, data, and code for security threats.
588
- """
589
- config = ConfigLoader.load()
596
+ """Scans models, data, and code for security threats."""
597
+ # Work on a shallow copy to prevent mutating the singleton between runs
598
+ base_config = ConfigLoader.load()
599
+ config = copy.copy(base_config)
600
+
590
601
  if verbose: logger.setLevel(logging.DEBUG)
591
602
  is_machine_output = (json_output or sarif_output or sbom_output) and not output_file
592
603
 
593
- # Passing the flags from the console to the global config object
594
604
  config.report_url = report_to or config.report_url
595
605
  config.api_key = api_key or config.api_key
596
606
 
@@ -637,7 +647,6 @@ def scan(
637
647
  console.print(Panel.fit(f"🛡️ [bold cyan]Veritensor Security Scanner[/bold cyan] v{__version__}", border_style="cyan"))
638
648
 
639
649
  if watch:
640
- import time
641
650
  console.print("[bold cyan]👀 Veritensor Watcher started. Monitoring files for changes... (Press Ctrl+C to stop)[/bold cyan]")
642
651
 
643
652
  # Collecting the initial file modification dates
@@ -921,7 +930,10 @@ def _print_table(results: List[ScanResult]):
921
930
  else:
922
931
  unique_threats = list(dict.fromkeys(res.threats))
923
932
  display_threats = "\n".join(unique_threats)
924
- table.add_row(res.file_path.split("/")[-1], f"[{status_style}]{res.status}[/{status_style}]", display_threats)
933
+
934
+ # Use Path.name for cross-platform compatibility
935
+ file_name = Path(res.file_path).name if res.file_path else "unknown"
936
+ table.add_row(file_name, f"[{status_style}]{res.status}[/{status_style}]", display_threats)
925
937
  console.print(table)
926
938
 
927
939
  def _perform_signing(image: str, status: str, config, timestamp: str, results: List[ScanResult]):
@@ -20,8 +20,11 @@ class HashCache:
20
20
  if not self.conn:
21
21
  return
22
22
  try:
23
- cutoff_mtime = time.time() - (days * 86400)
24
- self.cursor.execute("DELETE FROM file_cache WHERE mtime < ?", (cutoff_mtime,))
23
+ cutoff = time.time() - (days * 86400)
24
+ self.cursor.execute(
25
+ "DELETE FROM file_cache WHERE COALESCE(last_scanned, mtime) < ?",
26
+ (cutoff,)
27
+ )
25
28
  self.conn.commit()
26
29
  except Exception as e:
27
30
  logger.debug(f"Cache cleanup failed: {e}")
@@ -35,11 +38,9 @@ class HashCache:
35
38
  # check_same_thread=False allows using connection across threads.
36
39
  # In our multiprocessing architecture, only the MAIN process interacts with this class.
37
40
  self.conn = sqlite3.connect(str(CACHE_FILE), check_same_thread=False)
38
-
39
41
  # Enable WAL mode for better performance and concurrency safety
40
42
  self.conn.execute("PRAGMA journal_mode=WAL;")
41
43
  self.conn.execute("PRAGMA synchronous=NORMAL;")
42
-
43
44
  self.cursor = self.conn.cursor()
44
45
 
45
46
  # Create table
@@ -48,14 +49,24 @@ class HashCache:
48
49
  path TEXT PRIMARY KEY,
49
50
  hash TEXT,
50
51
  size INTEGER,
51
- mtime REAL
52
+ mtime REAL,
53
+ last_scanned REAL
52
54
  )
53
55
  """)
56
+
57
+ # Handle migration for existing DBs
58
+ try:
59
+ self.cursor.execute("ALTER TABLE file_cache ADD COLUMN last_scanned REAL")
60
+ self.conn.commit()
61
+ except Exception:
62
+ pass # Column already exists
63
+
54
64
  self.conn.commit()
55
65
  except Exception as e:
56
66
  logger.warning(f"Failed to initialize cache DB: {e}")
57
67
  self.conn = None
58
68
 
69
+
59
70
  def get(self, file_path: Path) -> Optional[str]:
60
71
  """Returns the hash if the file has not been changed."""
61
72
  if not self.conn: return None
@@ -71,16 +82,21 @@ class HashCache:
71
82
  row = self.cursor.fetchone()
72
83
 
73
84
  if row:
74
- cached_hash, cached_size, cached_mtime = row
75
- # Compare size and mtime (float comparison usually safe for exact system mtime)
85
+ cached_hash, cached_size, cached_mtime = row[:3]
76
86
  if cached_size == stats.st_size and cached_mtime == stats.st_mtime:
87
+ # Update last_scanned on cache hit
88
+ self.cursor.execute(
89
+ "UPDATE file_cache SET last_scanned = ? WHERE path = ?",
90
+ (time.time(), abs_path)
91
+ )
92
+ self.conn.commit()
77
93
  return cached_hash
78
94
 
79
95
  return None
80
96
  except Exception:
81
- # If file not found or permission error during stat
82
97
  return None
83
98
 
99
+
84
100
  def set(self, file_path: Path, file_hash: str):
85
101
  """Saves or updates the hash in the cache."""
86
102
  if not self.conn: return
@@ -90,9 +106,9 @@ class HashCache:
90
106
  stats = file_path.stat()
91
107
 
92
108
  self.cursor.execute("""
93
- INSERT OR REPLACE INTO file_cache (path, hash, size, mtime)
94
- VALUES (?, ?, ?, ?)
95
- """, (abs_path, file_hash, stats.st_size, stats.st_mtime))
109
+ INSERT OR REPLACE INTO file_cache (path, hash, size, mtime, last_scanned)
110
+ VALUES (?, ?, ?, ?, ?)
111
+ """, (abs_path, file_hash, stats.st_size, stats.st_mtime, time.time()))
96
112
 
97
113
  self.conn.commit()
98
114
  except Exception as e:
@@ -0,0 +1,67 @@
1
+ # Copyright 2026 Veritensor Security Apache 2.0
2
+ import socket
3
+ import ipaddress
4
+ import logging
5
+ import urllib3.util.connection as urllib3_cn
6
+ import requests
7
+
8
+ logger = logging.getLogger(__name__)
9
+
10
+ # Save the original connection function
11
+ _orig_create_connection = urllib3_cn.create_connection
12
+
13
+ def validate_ip_ssrf(ip_addr: str) -> bool:
14
+ """Checks if an IP address is public and safe to connect to."""
15
+ try:
16
+ ip_obj = ipaddress.ip_address(ip_addr)
17
+ if (ip_obj.is_private or ip_obj.is_loopback or
18
+ ip_obj.is_link_local or ip_obj.is_multicast or
19
+ ip_obj.is_reserved or ip_obj.is_unspecified):
20
+ return False
21
+ return True
22
+ except ValueError:
23
+ return False
24
+
25
+ def safe_create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, socket_options=None):
26
+ """
27
+ Patched connection creator. Resolves DNS, validates against SSRF,
28
+ and forces the socket to connect to the safe IP, preventing TOCTOU DNS Rebinding.
29
+ TLS/SNI remains intact because urllib3 wraps the socket with the original hostname later.
30
+ """
31
+ host, port = address
32
+
33
+ try:
34
+ ip_list = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
35
+ except socket.gaierror:
36
+ raise ValueError(f"Could not resolve hostname: {host}")
37
+
38
+ safe_ip = None
39
+ for item in ip_list:
40
+ ip_addr = item[4][0]
41
+ if validate_ip_ssrf(ip_addr):
42
+ safe_ip = ip_addr
43
+ break
44
+
45
+ if not safe_ip:
46
+ raise ValueError(f"SSRF Protection: No safe public IP found for {host}. Access forbidden.")
47
+
48
+ # Connect to the resolved and validated IP
49
+ return _orig_create_connection((safe_ip, port), timeout, source_address, socket_options)
50
+
51
+ # Apply the patch globally for the CLI runtime
52
+ urllib3_cn.create_connection = safe_create_connection
53
+
54
+ def get_safe_session(url: str = None) -> requests.Session:
55
+ """
56
+ Returns an SSRF-protected session.
57
+ The global urllib3 patch ensures all requests made by this session are safe.
58
+ """
59
+ if url:
60
+ # Trigger a dry-run resolution check immediately if a URL is provided
61
+ from urllib.parse import urlparse
62
+ parsed = urlparse(url)
63
+ hostname = parsed.hostname
64
+ if hostname:
65
+ safe_create_connection((hostname, parsed.port or 80))
66
+
67
+ return requests.Session()
@@ -1,4 +1,3 @@
1
-
2
1
  import zipfile
3
2
  import logging
4
3
 
@@ -32,9 +31,9 @@ class SafeZipReader:
32
31
 
33
32
  total_size += info.file_size
34
33
 
35
- # Checking the total size
36
- if total_size > SafeZipReader.MAX_UNZIPPED_SIZE:
37
- raise ZipBombError(f"Zip Bomb detected! Total unzipped size {total_size} bytes exceeds limit.")
34
+ # Checking the total size
35
+ if total_size > SafeZipReader.MAX_UNZIPPED_SIZE:
36
+ raise ZipBombError(f"Zip Bomb detected! Total unzipped size {total_size} bytes exceeds limit.")
38
37
 
39
38
  def read(self, zfile: zipfile.ZipFile, name: str) -> bytes:
40
39
  info = zfile.getinfo(name)
@@ -29,7 +29,7 @@ class RemoteStream(io.IOBase):
29
29
  def __init__(self, url: str, session: Optional[requests.Session] = None):
30
30
  self._validate_url(url)
31
31
  self.url = url
32
- self.session = session or requests.Session()
32
+ self._session = session or get_safe_session(self.url)
33
33
  self.pos = 0
34
34
  self.size = self._fetch_size()
35
35
  self._closed = False
@@ -60,10 +60,9 @@ class RemoteStream(io.IOBase):
60
60
  def _fetch_size(self) -> int:
61
61
  try:
62
62
  headers = {"Range": "bytes=0-0"}
63
- with get_safe_session(self.url):
64
- resp = self.session.get(self.url, headers=headers, stream=True, timeout=10)
65
- resp.raise_for_status()
66
-
63
+ resp = self._session.get(self.url, headers=headers, stream=True, timeout=10)
64
+ resp.raise_for_status()
65
+
67
66
  content_range = resp.headers.get("Content-Range")
68
67
  if content_range and "/" in content_range:
69
68
  return int(content_range.split("/")[-1])
@@ -76,6 +75,7 @@ class RemoteStream(io.IOBase):
76
75
  logger.error(f"Failed to fetch size for {self.url}: {e}")
77
76
  raise
78
77
 
78
+
79
79
  def read(self, size: int = -1) -> bytes:
80
80
  if self._closed: raise ValueError("I/O operation on closed file.")
81
81
  if self.pos >= self.size: return b""
@@ -89,9 +89,9 @@ class RemoteStream(io.IOBase):
89
89
 
90
90
  headers = {"Range": f"bytes={self.pos}-{end}"}
91
91
  try:
92
- with get_safe_session(self.url):
93
- resp = self.session.get(self.url, headers=headers, timeout=30)
94
- resp.raise_for_status()
92
+ # Use safe session
93
+ resp = self._session.get(self.url, headers=headers, timeout=30)
94
+ resp.raise_for_status()
95
95
  data = resp.content
96
96
  self.pos += len(data)
97
97
  return data
@@ -111,7 +111,12 @@ class RemoteStream(io.IOBase):
111
111
  def tell(self) -> int: return self.pos
112
112
  def seekable(self) -> bool: return True
113
113
  def readable(self) -> bool: return True
114
- def close(self): self._closed = True
114
+ def close(self):
115
+ # Properly close the session and call super().close()
116
+ if not self._closed:
117
+ self._closed = True
118
+ self._session.close()
119
+ super().close()
115
120
  def __enter__(self): return self
116
121
  def __exit__(self, exc_type, exc, tb): self.close()
117
122
 
@@ -189,7 +194,11 @@ class S3Stream(io.IOBase):
189
194
  def tell(self) -> int: return self.pos
190
195
  def seekable(self) -> bool: return True
191
196
  def readable(self) -> bool: return True
192
- def close(self): self._closed = True
197
+ def close(self):
198
+ # Call super().close() to properly update io.IOBase state
199
+ if not self._closed:
200
+ self._closed = True
201
+ super().close()
193
202
  def __enter__(self): return self
194
203
  def __exit__(self, exc_type, exc, tb): self.close()
195
204
 
@@ -1,12 +1,5 @@
1
1
  from dataclasses import dataclass, field
2
2
  from typing import List, Optional
3
- from enum import Enum
4
-
5
- class Severity(str, Enum):
6
- CRITICAL = "CRITICAL"
7
- HIGH = "HIGH"
8
- MEDIUM = "MEDIUM"
9
- LOW = "LOW"
10
3
 
11
4
  @dataclass
12
5
  class ScanResult:
@@ -19,7 +19,7 @@ class PIIScanner:
19
19
 
20
20
  if PRESIDIO_AVAILABLE is None:
21
21
  try:
22
- # CRITICAL FIX: Lazy import inside the method to prevent DLL/OSError crashes on startup
22
+ # Lazy import inside the method to prevent DLL/OSError crashes on startup
23
23
  from presidio_analyzer import AnalyzerEngine
24
24
  from presidio_analyzer.nlp_engine import NlpEngineProvider
25
25
  import spacy
@@ -1,6 +1,6 @@
1
1
  # Copyright 2026 Veritensor Security Apache 2.0
2
2
  # Dataset Scanner (Parquet, CSV, JSONL) for Data Poisoning, Malicious URLs, and Toxic Columns
3
-
3
+ import threading
4
4
  import logging
5
5
  import json
6
6
  from pathlib import Path
@@ -31,23 +31,28 @@ FALLBACK_SUSPICIOUS = [
31
31
  "regex:(?i)secret"
32
32
  ]
33
33
 
34
- # --- CRITICAL FIX: Robust Singleton for GLiNER ---
34
+ # Thread-safe lazy loading for GLiNER
35
35
  _GLINER_UNAVAILABLE = object()
36
36
  _gliner_model = None
37
+ _gliner_lock = threading.Lock()
37
38
 
38
39
  def _get_gliner_model():
39
- """Lazy loads GLiNER to avoid 30-90s delay on every function call."""
40
40
  global _gliner_model
41
- if _gliner_model is None:
42
- try:
43
- from gliner import GLiNER
44
- logger.info("Loading GLiNER model for column analysis...")
45
- _gliner_model = GLiNER.from_pretrained("urchade/gliner_multi-v2.1")
46
- except ImportError:
47
- _gliner_model = _GLINER_UNAVAILABLE
48
- except Exception as e:
49
- logger.warning(f"GLiNER failed to load: {e}. Falling back to regex.")
50
- _gliner_model = _GLINER_UNAVAILABLE
41
+ if _gliner_model is not None:
42
+ return _gliner_model
43
+
44
+ with _gliner_lock:
45
+ if _gliner_model is None: # Double-checked locking
46
+ try:
47
+ from gliner import GLiNER
48
+ logger.info("Loading GLiNER model for column analysis...")
49
+ _gliner_model = GLiNER.from_pretrained("urchade/gliner_multi-v2.1")
50
+ except ImportError:
51
+ _gliner_model = _GLINER_UNAVAILABLE
52
+ except Exception as e:
53
+ logger.warning(f"GLiNER failed to load: {e}. Falling back to regex.")
54
+ _gliner_model = _GLINER_UNAVAILABLE
55
+
51
56
  return _gliner_model
52
57
 
53
58
  def _check_toxic_columns(columns: List[str]) -> List[str]:
@@ -60,7 +65,8 @@ def _check_toxic_columns(columns: List[str]) -> List[str]:
60
65
 
61
66
  toxic_keywords = [
62
67
  "social_credit", "political_affiliation", "sexual_orientation",
63
- "race", "religion", "trade_union", "biometric_categorization", "criminal_prediction"
68
+ "race", "religion", "trade_union", "biometric_categorization", "criminal_prediction",
69
+ "emotion_state", "sentiment_analysis_workplace", "facial_features", "cctv_face_id", "subliminal_metrics"
64
70
  ]
65
71
 
66
72
  model = _get_gliner_model()
@@ -95,15 +101,19 @@ def scan_dataset(file_path: Path, full_scan: bool = False, bias_profile: Optiona
95
101
 
96
102
  row_limit = None if full_scan else MAX_ROWS_DEFAULT
97
103
 
98
- # Initialize Bias Aggregator if profile is provided
99
- aggregator = BiasAggregator(bias_profile) if bias_profile else None
104
+ aggregator = None
105
+ if bias_profile:
106
+ try:
107
+ aggregator = BiasAggregator(bias_profile)
108
+ except ValueError as e:
109
+ logger.warning(f"Invalid bias profile for {file_path.name}: {e}")
100
110
 
101
111
  try:
102
112
  text_stream = None
103
113
 
104
114
  if ext == ".parquet":
105
115
  if not PYARROW_AVAILABLE:
106
- return ["WARNING: pyarrow not installed. Run 'pip install veritensor[data]'"], None
116
+ return ["WARNING: pyarrow and pandas are required for local parquet scanning. Run 'pip install pyarrow pandas' or use the Enterprise Server."], None
107
117
 
108
118
  try:
109
119
  parquet_file = pq.ParquetFile(file_path)
@@ -268,9 +278,16 @@ def _extract_strings_from_json(data: Any) -> Generator[str, None, None]:
268
278
  class BiasAggregator:
269
279
  """Aggregates counts for fairness metrics using O(1) memory. Safe for multiprocessing (no lambdas)."""
270
280
  def __init__(self, profile: dict):
271
- self.protected_attrs = profile.get("protected_attributes", [])
281
+ # Validate required fields to prevent AttributeError during processing
272
282
  self.target_var = profile.get("target_variable")
273
- self.favorable_label = str(profile.get("favorable_label"))
283
+ if not self.target_var:
284
+ raise ValueError("BiasAggregator requires 'target_variable' in bias profile")
285
+
286
+ self.protected_attrs = profile.get("protected_attributes", [])
287
+ if not self.protected_attrs:
288
+ raise ValueError("BiasAggregator requires 'protected_attributes' in bias profile")
289
+
290
+ self.favorable_label = str(profile.get("favorable_label", "1"))
274
291
 
275
292
  self.group_target_counts = {}
276
293
  self.proxy_counts = {}
@@ -24,7 +24,7 @@ FORMULA_PREFIXES = ('=', '+', '-', '@')
24
24
  def scan_excel(file_path: Path) -> List[str]:
25
25
  threats = []
26
26
  if not EXCEL_AVAILABLE:
27
- return ["WARNING: openpyxl not installed. Run 'pip install veritensor[data]'"]
27
+ return ["WARNING: openpyxl is required for local Excel scanning. Run 'pip install openpyxl' or use the Enterprise Server."]
28
28
 
29
29
  try:
30
30
  # read_only=True is faster and uses less memory
@@ -20,7 +20,11 @@ try:
20
20
  # Python 3.11+ native TOML support
21
21
  import tomllib
22
22
  except ImportError:
23
- tomllib = None
23
+ try:
24
+ # Backport for Python 3.10
25
+ import tomli as tomllib
26
+ except ImportError:
27
+ tomllib = None
24
28
 
25
29
  # --- Known Malware DB (MVP) ---
26
30
  KNOWN_MALICIOUS = {
@@ -214,9 +218,15 @@ def _parse_requirements(path: Path) -> Dict[str, Optional[str]]:
214
218
  if not line or line.startswith(("#", "-")):
215
219
  continue
216
220
 
221
+ # Strip inline comments before any further parsing
222
+ comment_idx = line.find(' #')
223
+ if comment_idx != -1:
224
+ line = line[:comment_idx].strip()
225
+ if not line:
226
+ continue
227
+
217
228
  # Processing direct links (git+, http) with #egg=
218
229
  if "egg=" in line:
219
- # Пример: git+https://github.com/user/repo.git#egg=evil_pkg
220
230
  name_part = line.split("egg=")[-1].split("&")[0].strip()
221
231
  deps[name_part] = None
222
232
  continue
@@ -236,6 +246,7 @@ def _parse_requirements(path: Path) -> Dict[str, Optional[str]]:
236
246
  return deps
237
247
 
238
248
 
249
+
239
250
  def _parse_pyproject(path: Path) -> Dict[str, Optional[str]]:
240
251
  deps = {}
241
252
  try:
@@ -131,7 +131,6 @@ _SEVERITY_MAP = {
131
131
  # Human-in-the-loop parameter names that indicate the tool has a confirmation gate
132
132
  _HITL_PARAM_NAMES = frozenset({"confirm", "approved", "dry_run", "dryrun", "force"})
133
133
 
134
- _STDLIB_ENTRYPOINTS = frozenset({"main", "__main__", "cli", "run", "start", "app"})
135
134
  # ---------------------------------------------------------------------------
136
135
  # Result types
137
136
  # ---------------------------------------------------------------------------
@@ -216,13 +215,22 @@ class _MCPToolVisitor(ast.NodeVisitor):
216
215
 
217
216
 
218
217
  def _is_mcp_tool(self, node: ast.FunctionDef) -> bool:
219
- if node.name.lower() in _STDLIB_ENTRYPOINTS:
220
- return False
218
+ # Check decorator FIRST. If @tool is present, scan regardless of name.
219
+ has_mcp_decorator = False
221
220
  for dec in node.decorator_list:
222
221
  name = _decorator_base_name(dec)
223
222
  if name and name.lower() in _MCP_TOOL_DECORATORS:
224
- return True
225
- return False
223
+ has_mcp_decorator = True
224
+ break
225
+
226
+ if not has_mcp_decorator:
227
+ return False
228
+
229
+ # Only skip dunder names (e.g., __init__) — they are never MCP tools
230
+ if node.name.startswith("__") and node.name.endswith("__"):
231
+ return False
232
+
233
+ return True
226
234
 
227
235
  def _has_hitl_param(self, node: ast.FunctionDef) -> bool:
228
236
  params = {arg.arg.lower() for arg in node.args.args}
@@ -10,6 +10,8 @@ import re
10
10
  from veritensor.core.entropy import is_high_entropy
11
11
  from veritensor.engines.static.rules import get_severity, SignatureLoader, is_match
12
12
  from veritensor.engines.content.pii import PIIScanner
13
+ import tokenize
14
+ import io as _io
13
15
 
14
16
  logger = logging.getLogger(__name__)
15
17
 
@@ -140,15 +142,21 @@ def _clean_magics(source: str) -> str:
140
142
  return "\n".join(lines)
141
143
 
142
144
  def _check_nesting_level(code: str, max_depth: int = 100) -> bool:
143
- # Protection against AST Bomb (Stack Overflow)
145
+ """Check bracket depth, ignoring content inside string literals."""
144
146
  depth = 0
145
- for char in code:
146
- if char in "([{":
147
- depth += 1
148
- if depth > max_depth:
149
- return False
150
- elif char in ")]}":
151
- depth -= 1
147
+ try:
148
+ tokens = tokenize.generate_tokens(_io.StringIO(code).readline)
149
+ for tok_type, tok_string, _, _, _ in tokens:
150
+ if tok_type == tokenize.OP:
151
+ if tok_string in ('(', '[', '{'):
152
+ depth += 1
153
+ if depth > max_depth:
154
+ return False
155
+ elif tok_string in (')', ']', '}'):
156
+ depth = max(0, depth - 1)
157
+ except tokenize.TokenError:
158
+ # Incomplete code (common in notebooks) — allow it to proceed to AST
159
+ pass
152
160
  return True
153
161
 
154
162