veritensor 1.9.1__tar.gz → 1.9.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {veritensor-1.9.1/src/veritensor.egg-info → veritensor-1.9.3}/PKG-INFO +10 -33
  2. {veritensor-1.9.1 → veritensor-1.9.3}/README.md +2 -28
  3. {veritensor-1.9.1 → veritensor-1.9.3}/pyproject.toml +9 -7
  4. veritensor-1.9.3/src/veritensor/__init__.py +2 -0
  5. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/cli/main.py +100 -90
  6. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/cache.py +28 -12
  7. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/config.py +0 -5
  8. veritensor-1.9.3/src/veritensor/core/networking.py +84 -0
  9. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/safe_zip.py +7 -6
  10. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/streaming.py +40 -17
  11. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/types.py +0 -7
  12. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/content/injection.py +1 -1
  13. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/content/pii.py +9 -2
  14. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/data/dataset_engine.py +46 -25
  15. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/data/excel_engine.py +14 -11
  16. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/calculator.py +12 -3
  17. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/lfs.py +11 -9
  18. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/readers.py +24 -0
  19. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/dependency_engine.py +23 -5
  20. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/mcp_scanner.py +13 -5
  21. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/notebook_engine.py +23 -11
  22. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/pickle_engine.py +2 -4
  23. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/rules.py +20 -14
  24. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/chroma_guard.py +2 -1
  25. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/enterprise_scanner.py +47 -30
  26. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/huggingface.py +42 -33
  27. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/rag.py +12 -3
  28. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/unstructured_guard.py +2 -1
  29. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/compliance_report.py +25 -12
  30. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/html_report.py +32 -11
  31. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/sarif.py +30 -9
  32. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/sbom.py +5 -1
  33. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/telemetry.py +14 -2
  34. {veritensor-1.9.1 → veritensor-1.9.3/src/veritensor.egg-info}/PKG-INFO +10 -33
  35. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/SOURCES.txt +0 -1
  36. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/requires.txt +9 -4
  37. veritensor-1.9.1/src/veritensor/__init__.py +0 -2
  38. veritensor-1.9.1/src/veritensor/core/networking.py +0 -64
  39. veritensor-1.9.1/src/veritensor/integrations/cosign.py +0 -105
  40. {veritensor-1.9.1 → veritensor-1.9.3}/LICENSE +0 -0
  41. {veritensor-1.9.1 → veritensor-1.9.3}/MANIFEST.in +0 -0
  42. {veritensor-1.9.1 → veritensor-1.9.3}/setup.cfg +0 -0
  43. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/__main__.py +0 -0
  44. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/cli/__init__.py +0 -0
  45. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/__init__.py +0 -0
  46. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/entropy.py +0 -0
  47. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/file_utils.py +0 -0
  48. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/core/text_utils.py +0 -0
  49. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/__init__.py +0 -0
  50. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/container/__init__.py +0 -0
  51. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/container/archive_engine.py +0 -0
  52. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/content/__init__.py +0 -0
  53. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/data/__init__.py +0 -0
  54. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/hashing/__init__.py +0 -0
  55. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/__init__.py +0 -0
  56. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/keras_engine.py +0 -0
  57. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/mcp_permission_auditor.py +0 -0
  58. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/engines/static/signatures.yaml +0 -0
  59. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/__init__.py +0 -0
  60. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/langchain_guard.py +0 -0
  61. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/integrations/llamaindex_guard.py +0 -0
  62. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/excel_report.py +0 -0
  63. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor/reporting/manifest.py +0 -0
  64. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/dependency_links.txt +0 -0
  65. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/entry_points.txt +0 -0
  66. {veritensor-1.9.1 → veritensor-1.9.3}/src/veritensor.egg-info/top_level.txt +0 -0
  67. {veritensor-1.9.1 → veritensor-1.9.3}/veritensor.yaml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: veritensor
3
- Version: 1.9.1
3
+ Version: 1.9.3
4
4
  Summary: Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats.
5
5
  Author-email: Arsenii Brazhnyk <veritensor@gmail.com>
6
6
  License: Apache-2.0
@@ -28,13 +28,16 @@ Requires-Dist: pyyaml>=6.0
28
28
  Requires-Dist: huggingface_hub>=0.19.0
29
29
  Requires-Dist: cyclonedx-python-lib>=6.0.0
30
30
  Requires-Dist: jinja2>=3.1.0
31
- Requires-Dist: pillow>=12.2.0
31
+ Requires-Dist: pillow>=12.3.0
32
32
  Requires-Dist: lxml>=6.1.0
33
33
  Requires-Dist: urllib3>=2.7.0
34
- Requires-Dist: gitpython>=3.1.49
34
+ Requires-Dist: gitpython>=3.1.54
35
35
  Requires-Dist: pyarrow>=14.0.0
36
36
  Requires-Dist: pandas>=2.0.0
37
37
  Requires-Dist: openpyxl>=3.1.0
38
+ Requires-Dist: defusedxml>=0.7.1
39
+ Requires-Dist: tomli>=2.0.0; python_version < "3.11"
40
+ Requires-Dist: msgpack>=1.2.1
38
41
  Provides-Extra: test
39
42
  Requires-Dist: pytest>=7.0.0; extra == "test"
40
43
  Requires-Dist: pytest-mock>=3.10.0; extra == "test"
@@ -46,14 +49,14 @@ Provides-Extra: pii
46
49
  Requires-Dist: presidio-analyzer>=2.2.0; extra == "pii"
47
50
  Requires-Dist: spacy>=3.0.0; extra == "pii"
48
51
  Provides-Extra: rag
49
- Requires-Dist: pypdf>=4.0.0; extra == "rag"
52
+ Requires-Dist: pypdf>=6.14.2; extra == "rag"
50
53
  Requires-Dist: python-docx>=1.1.0; extra == "rag"
51
54
  Requires-Dist: python-pptx>=0.6.21; extra == "rag"
52
55
  Requires-Dist: veritensor[pii]; extra == "rag"
53
56
  Provides-Extra: aws
54
57
  Requires-Dist: boto3>=1.28.0; extra == "aws"
55
58
  Provides-Extra: all
56
- Requires-Dist: veritensor[aws,data,pii,rag]; extra == "all"
59
+ Requires-Dist: veritensor[aws,pii,rag]; extra == "all"
57
60
  Provides-Extra: dev
58
61
  Requires-Dist: veritensor[all,test]; extra == "dev"
59
62
  Requires-Dist: black; extra == "dev"
@@ -395,32 +398,6 @@ veritensor scan ./models \
395
398
  --sarif --output-file report.sarif \
396
399
  --json --output-file results.json
397
400
  ```
398
- ---
399
-
400
- ## 🔐 Supply Chain Security (Container Signing)
401
-
402
- Veritensor integrates with Sigstore Cosign to cryptographically sign your Docker images only if they pass the security scan.
403
-
404
- ### 1. Generate Keys
405
- Generate a key pair for signing:
406
- ```bash
407
- veritensor keygen
408
- # Output: veritensor.key (Private) and veritensor.pub (Public)
409
- ```
410
- ### 2. Scan & Sign
411
- Pass the --image flag and the path to your private key (via env var).
412
- ```bash
413
- # Set path to your private key
414
- export VERITENSOR_PRIVATE_KEY_PATH=veritensor.key
415
-
416
- # If scan passes -> Sign the image
417
- veritensor scan ./models/my_model.pkl --image my-org/my-app:v1.0.0
418
- ```
419
- ### 3. Verify (In Kubernetes / Production)
420
- Before deploying, verify the signature to ensure the model was scanned:
421
- ```bash
422
- cosign verify --key veritensor.pub my-org/my-app:v1.0.0
423
- ```
424
401
 
425
402
  ---
426
403
 
@@ -443,7 +420,7 @@ jobs:
443
420
  steps:
444
421
  - uses: actions/checkout@v4
445
422
  - name: Veritensor Scan
446
- uses: arsbr/Veritensor@v1.9.1
423
+ uses: arsbr/Veritensor@v1.9.3
447
424
  with:
448
425
  path: '.'
449
426
  jobs: '4'
@@ -454,7 +431,7 @@ Prevent committing malicious models to your repository. Add this to .pre-commit-
454
431
  ```yaml
455
432
  repos:
456
433
  - repo: https://github.com/arsbr/Veritensor
457
- rev: v1.9.1
434
+ rev: v1.9.3
458
435
  hooks:
459
436
  - id: veritensor-scan
460
437
  ```
@@ -333,32 +333,6 @@ veritensor scan ./models \
333
333
  --sarif --output-file report.sarif \
334
334
  --json --output-file results.json
335
335
  ```
336
- ---
337
-
338
- ## 🔐 Supply Chain Security (Container Signing)
339
-
340
- Veritensor integrates with Sigstore Cosign to cryptographically sign your Docker images only if they pass the security scan.
341
-
342
- ### 1. Generate Keys
343
- Generate a key pair for signing:
344
- ```bash
345
- veritensor keygen
346
- # Output: veritensor.key (Private) and veritensor.pub (Public)
347
- ```
348
- ### 2. Scan & Sign
349
- Pass the --image flag and the path to your private key (via env var).
350
- ```bash
351
- # Set path to your private key
352
- export VERITENSOR_PRIVATE_KEY_PATH=veritensor.key
353
-
354
- # If scan passes -> Sign the image
355
- veritensor scan ./models/my_model.pkl --image my-org/my-app:v1.0.0
356
- ```
357
- ### 3. Verify (In Kubernetes / Production)
358
- Before deploying, verify the signature to ensure the model was scanned:
359
- ```bash
360
- cosign verify --key veritensor.pub my-org/my-app:v1.0.0
361
- ```
362
336
 
363
337
  ---
364
338
 
@@ -381,7 +355,7 @@ jobs:
381
355
  steps:
382
356
  - uses: actions/checkout@v4
383
357
  - name: Veritensor Scan
384
- uses: arsbr/Veritensor@v1.9.1
358
+ uses: arsbr/Veritensor@v1.9.3
385
359
  with:
386
360
  path: '.'
387
361
  jobs: '4'
@@ -392,7 +366,7 @@ Prevent committing malicious models to your repository. Add this to .pre-commit-
392
366
  ```yaml
393
367
  repos:
394
368
  - repo: https://github.com/arsbr/Veritensor
395
- rev: v1.9.1
369
+ rev: v1.9.3
396
370
  hooks:
397
371
  - id: veritensor-scan
398
372
  ```
@@ -1,10 +1,10 @@
1
1
  [build-system]
2
- requires = ["setuptools>=61.0"]
2
+ requires = ["setuptools>=78.1.1", "wheel>=0.46.2"]
3
3
  build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "veritensor"
7
- version = "1.9.1"
7
+ version = "1.9.3"
8
8
  description = "Antivirus for the AI Supply Chain. Scans models, datasets, notebooks, and RAG documents for threats."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -33,14 +33,16 @@ dependencies = [
33
33
  "huggingface_hub>=0.19.0",
34
34
  "cyclonedx-python-lib>=6.0.0",
35
35
  "jinja2>=3.1.0",
36
- "pillow>=12.2.0",
36
+ "pillow>=12.3.0",
37
37
  "lxml>=6.1.0",
38
38
  "urllib3 >= 2.7.0",
39
- # Fickling removed from here to avoid installing PyTorch (800MB+) for regular users
40
- "gitpython>=3.1.49",
39
+ "gitpython>=3.1.54",
41
40
  "pyarrow>=14.0.0",
42
41
  "pandas>=2.0.0",
43
42
  "openpyxl>=3.1.0",
43
+ "defusedxml>=0.7.1",
44
+ "tomli>=2.0.0; python_version < '3.11'",
45
+ "msgpack>=1.2.1",
44
46
  ]
45
47
 
46
48
 
@@ -63,7 +65,7 @@ pii = [
63
65
 
64
66
  # Optional dependencies for RAG document scanning (PDF, DOCX)
65
67
  rag = [
66
- "pypdf>=4.0.0",
68
+ "pypdf>=6.14.2",
67
69
  "python-docx>=1.1.0",
68
70
  "python-pptx>=0.6.21",
69
71
  "veritensor[pii]"
@@ -75,7 +77,7 @@ aws = [
75
77
  ]
76
78
 
77
79
  all = [
78
- "veritensor[aws,data,rag,pii]"
80
+ "veritensor[aws,rag,pii]"
79
81
  ]
80
82
 
81
83
  dev = [
@@ -0,0 +1,2 @@
1
+ __version__ = "1.9.3"
2
+
@@ -11,6 +11,7 @@ import os
11
11
  import datetime
12
12
  import time
13
13
  import requests
14
+ import copy
14
15
  import fnmatch
15
16
  import concurrent.futures
16
17
  import multiprocessing
@@ -60,7 +61,6 @@ except ImportError:
60
61
  from veritensor.engines.static.rules import is_license_restricted
61
62
  def is_match(repo, allowed): return False
62
63
 
63
- from veritensor.integrations.cosign import sign_container, is_cosign_available, generate_key_pair
64
64
  from veritensor.integrations.huggingface import HuggingFaceClient
65
65
 
66
66
  logging.basicConfig(level=logging.INFO, format="%(message)s")
@@ -109,7 +109,7 @@ HEAVY_EXTS = {
109
109
  # --- AI Notebooks (AST, Secrets, Outputs) ---
110
110
  ".ipynb",
111
111
 
112
- # --- Datasets (Сэмплируются перед отправкой!) ---
112
+ # --- Datasets (sampled before upload to reduce payload size!) ---
113
113
  ".parquet", ".csv", ".tsv", ".jsonl", ".ndjson",
114
114
 
115
115
  # --- Text & Markup (DeBERTa Semantic Scan, GLiNER) ---
@@ -131,6 +131,22 @@ HEAVY_EXTS = {
131
131
  # Specific files that are sent to the server (name verification)
132
132
  HEAVY_FILES = {"dockerfile"}
133
133
 
134
+ def _build_api_url(report_url: str, endpoint: str) -> str:
135
+ """
136
+ Safely constructs API endpoint URL.
137
+ Expected input formats:
138
+ - http://server/api/v1
139
+ - http://server/api/v1/telemetry
140
+ """
141
+ if not report_url:
142
+ return ""
143
+ base = report_url.rstrip("/")
144
+ for suffix in ("/api/v1/telemetry", "/telemetry", "/api/v1"):
145
+ if base.endswith(suffix):
146
+ base = base[: -len(suffix)]
147
+ break
148
+
149
+ return f"{base.rstrip('/')}/api/v1{endpoint}"
134
150
 
135
151
  def load_ignore_patterns(ignore_file: str = ".veritensorignore") -> List[str]:
136
152
  """Loads glob patterns from .veritensorignore file."""
@@ -166,7 +182,7 @@ def check_remote_cache(report_url: str, api_key: str, hashes: List[str], version
166
182
  if not report_url or not api_key or not hashes:
167
183
  return {}
168
184
 
169
- cache_url = report_url.replace("/telemetry", "/cache/check")
185
+ cache_url = _build_api_url(report_url, "/cache/check")
170
186
  headers = {"X-API-Key": api_key}
171
187
  payload = {"hashes": hashes, "scanner_version": version} # Added a version to the payload
172
188
 
@@ -183,7 +199,7 @@ def fetch_server_policies(report_url: str, api_key: str) -> Tuple[Optional[dict]
183
199
  if not report_url or not api_key:
184
200
  return None,[]
185
201
 
186
- policy_url = report_url.replace("/telemetry", "/policies")
202
+ policy_url = _build_api_url(report_url, "/policies")
187
203
  headers = {"X-API-Key": api_key}
188
204
 
189
205
  try:
@@ -204,7 +220,8 @@ def is_suppressed(file_path: str, threat_msg: str, suppressions: List[dict]) ->
204
220
  if not supp_path or not supp_type:
205
221
  continue
206
222
  file_name = Path(file_path).name
207
- if file_name == supp_path and supp_type in threat_msg:
223
+ supp_name = Path(supp_path).name
224
+ if file_name == supp_name and supp_type in threat_msg:
208
225
  return True
209
226
  return False
210
227
 
@@ -239,6 +256,16 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
239
256
  scan_res = ScanResult(file_path=file_path_str)
240
257
  scan_res.repo_id = repo
241
258
 
259
+ # Check MCP config files early, before extension-based routing swallows .json files
260
+ if file_path and ext == ".json" and not is_s3:
261
+ try:
262
+ if is_mcp_config_file(file_path):
263
+ perm_result = audit_mcp_config(file_path)
264
+ for t in perm_result.to_threat_strings():
265
+ scan_res.add_threat(t)
266
+ except Exception as e:
267
+ logger.debug(f"MCP Audit failed for {file_name}: {e}")
268
+
242
269
  # --- HYBRID ROUTING (Sending heavy files to the server) ---
243
270
  # If the user has an Enterprise server connected
244
271
  already_remote_scanned = False
@@ -252,12 +279,28 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
252
279
  scanner = EnterpriseScanner(config.report_url, config.api_key)
253
280
  remote_threats = scanner.scan_file_remotely(file_path, full_scan=full_scan_dataset)
254
281
 
255
- if remote_threats:
256
- for t in remote_threats: scan_res.add_threat(t)
282
+ # Detect network/timeout failures by inspecting returned warnings
283
+ _INFRA_FAILURE_PREFIXES = (
284
+ "WARNING: Failed to reach",
285
+ "WARNING: Enterprise scan timed out",
286
+ "WARNING: S3",
287
+ )
288
+ is_infra_failure = any(
289
+ t.startswith(_INFRA_FAILURE_PREFIXES) for t in remote_threats
290
+ ) if remote_threats else False
257
291
 
258
- already_remote_scanned = True
259
- # The file has been verified on the server (ML, OCR, YARA), so we skip the local checks
260
- #return scan_res
292
+ if is_infra_failure:
293
+ # Infrastructure failure: do NOT set already_remote_scanned
294
+ # so local fallback can run for supported formats
295
+ logger.warning(f"Remote scan unavailable for {file_name}. Falling back to local scan.")
296
+ for t in remote_threats:
297
+ scan_res.threats.append(t) # Append without triggering immediate FAIL status
298
+ else:
299
+ if remote_threats:
300
+ for t in remote_threats:
301
+ scan_res.add_threat(t)
302
+ already_remote_scanned = True
303
+
261
304
  except Exception as e:
262
305
  scan_res.add_threat(f"WARNING: Remote scan failed, falling back to local: {e}")
263
306
 
@@ -318,8 +361,7 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
318
361
  try:
319
362
 
320
363
  safe_filename = quote(file_name, safe='')
321
- prof_url = config.report_url.replace("/telemetry", f"/fairness/profiles/{safe_filename}")
322
-
364
+ prof_url = _build_api_url(config.report_url, f"/fairness/profiles/{safe_filename}")
323
365
  res = requests.get(prof_url, headers={"X-API-Key": config.api_key}, timeout=5)
324
366
  if res.status_code == 200:
325
367
  bias_profile = res.json().get("profile")
@@ -330,8 +372,9 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
330
372
  # Pass the profile to the scanner
331
373
  threats, bias_data = scan_dataset(file_path, full_scan=full_scan_dataset, bias_profile=bias_profile)
332
374
 
333
- for t in threats:
334
- scan_res.add_threat(t)
375
+ if not already_remote_scanned:
376
+ for t in threats:
377
+ scan_res.add_threat(t)
335
378
 
336
379
  # Attach bias data to the result so it gets sent in telemetry
337
380
  if bias_data:
@@ -360,12 +403,7 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
360
403
  if mcp_result.mcp_tools_found:
361
404
  for t in mcp_result.to_threat_strings():
362
405
  scan_res.add_threat(t)
363
- elif ext == ".json" and file_path:
364
- if is_mcp_config_file(file_path):
365
- from veritensor.engines.static.mcp_permission_auditor import audit_mcp_config
366
- perm_result = audit_mcp_config(file_path)
367
- for t in perm_result.to_threat_strings():
368
- scan_res.add_threat(t)
406
+
369
407
  else:
370
408
  # If the format is unknown to any engine at all.
371
409
  # We add INFO, but DO NOT call add_threat(),
@@ -379,11 +417,6 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
379
417
  except Exception as e:
380
418
  scan_res.add_threat(f"CRITICAL: Engine Error: {str(e)}")
381
419
 
382
- # --- MERGE REMOTE THREATS ---
383
- for t in remote_threats:
384
- if t not in scan_res.threats:
385
- scan_res.add_threat(t)
386
-
387
420
  # --- C. License Check ---
388
421
  if not is_s3 and file_path:
389
422
  reader = get_reader_for_file(file_path)
@@ -414,6 +447,11 @@ def scan_worker(args: Tuple[str, VeritensorConfig, Optional[str], bool, bool, bo
414
447
 
415
448
  return scan_res
416
449
 
450
+ def _worker_initializer():
451
+ """Pre-load spaCy model once per worker process to avoid RAM spikes during scans."""
452
+ from veritensor.engines.content.pii import PIIScanner
453
+ PIIScanner.get_engine()
454
+
417
455
  # --- SHARED SCAN LOGIC ---
418
456
  def _run_scan_process(
419
457
  paths: List[str], repo: Optional[str], jobs: Optional[int],
@@ -424,8 +462,8 @@ def _run_scan_process(
424
462
  Core logic to collect files and run parallel scan.
425
463
  Used by both 'scan' and 'manifest' commands.
426
464
  """
427
- files_to_scan =[]
428
- ignore_patterns = load_ignore_patterns() # Load .veritensorignore
465
+ files_to_scan: List[str] = []
466
+ ignore_patterns = load_ignore_patterns()
429
467
 
430
468
  # 1. Collect Files from all paths
431
469
  for path in paths:
@@ -436,11 +474,11 @@ def _run_scan_process(
436
474
  local_path = Path(path)
437
475
  if local_path.is_file():
438
476
  if not is_ignored(local_path, ignore_patterns):
439
- files_to_scan.append(local_path)
477
+ files_to_scan.append(str(local_path))
440
478
  elif local_path.is_dir():
441
479
  for p in local_path.rglob("*"):
442
480
  if p.is_file() and not is_ignored(p, ignore_patterns):
443
- files_to_scan.append(p)
481
+ files_to_scan.append(str(p))
444
482
  else:
445
483
  raise FileNotFoundError(f"Path {path} not found.")
446
484
 
@@ -460,21 +498,16 @@ def _run_scan_process(
460
498
  tasks = []
461
499
 
462
500
  # First, we quickly calculate the hashes locally (using SQLite so as not to read the files again)
463
- local_hashes_map = {} # {filepath: hash}
501
+ local_hashes_map = {}
502
+
464
503
  for f in files_to_scan:
465
- if str(f).startswith("s3://"):
504
+ if f.startswith("s3://"):
466
505
  continue
467
- cached_hash = hash_cache.get(f)
506
+
507
+ file_path_obj = Path(f)
508
+ cached_hash = hash_cache.get(file_path_obj)
468
509
  if cached_hash:
469
- local_hashes_map[str(f)] = cached_hash
470
- else:
471
- # If there is no local cache, you will have to calculate
472
- try:
473
- h = calculate_sha256(f)
474
- local_hashes_map[str(f)] = h
475
- hash_cache.set(f, h)
476
- except Exception:
477
- pass
510
+ local_hashes_map[f] = cached_hash
478
511
 
479
512
  # We ask the server about these hashes
480
513
  remote_cache_results = {}
@@ -486,17 +519,17 @@ def _run_scan_process(
486
519
 
487
520
  # Creating tasks
488
521
  for f in files_to_scan:
489
- is_s3 = str(f).startswith("s3://")
522
+ is_s3 = f.startswith("s3://")
490
523
  if is_s3:
491
- tasks.append((str(f), config, repo, ignore_license, full_scan, True, None))
524
+ tasks.append((f, config, repo, ignore_license, full_scan, True, None))
492
525
  continue
493
526
 
494
- file_hash = local_hashes_map.get(str(f))
527
+ file_hash = local_hashes_map.get(f)
495
528
 
496
529
  # If the server knows this file, we DON'T add it to tasks
497
530
  if file_hash and file_hash in remote_cache_results:
498
531
  remote_data = remote_cache_results[file_hash]
499
- res = ScanResult(file_path=str(f), status=remote_data["status"], file_hash=file_hash)
532
+ res = ScanResult(file_path=f, status=remote_data["status"], file_hash=file_hash)
500
533
  res.threats = remote_data["threats"]
501
534
 
502
535
  reader = get_reader_for_file(Path(f))
@@ -509,7 +542,7 @@ def _run_scan_process(
509
542
 
510
543
  results.append(res)
511
544
  else:
512
- tasks.append((str(f), config, repo, ignore_license, full_scan, False, file_hash))
545
+ tasks.append((f, config, repo, ignore_license, full_scan, False, file_hash))
513
546
 
514
547
  # 3. Execute
515
548
  executor = None
@@ -527,7 +560,10 @@ def _run_scan_process(
527
560
  ) as progress:
528
561
 
529
562
  main_task = progress.add_task("Scanning...", total=len(tasks))
530
- executor = concurrent.futures.ProcessPoolExecutor(max_workers=jobs)
563
+ executor = concurrent.futures.ProcessPoolExecutor(
564
+ max_workers=jobs,
565
+ initializer=_worker_initializer
566
+ )
531
567
 
532
568
  future_to_file = {
533
569
  executor.submit(scan_worker, task_args): task_args[0]
@@ -563,7 +599,6 @@ def _run_scan_process(
563
599
  def scan(
564
600
  paths: List[str] = typer.Argument(..., help="Paths to files, directories, or S3 URLs"),
565
601
  repo: Optional[str] = typer.Option(None, "--repo", "-r", help="Hugging Face Repo ID"),
566
- image: Optional[str] = typer.Option(None, help="Docker image tag to sign"),
567
602
  ignore_license: bool = typer.Option(False, "--ignore-license", help="Do not fail on license violations"),
568
603
  ignore_malware: bool = typer.Option(False, "--ignore-malware", help="Do not fail on malware/policy violations"),
569
604
  full_scan: bool = typer.Option(False, "--full-scan", help="Scan entire dataset (slow)."),
@@ -583,14 +618,14 @@ def scan(
583
618
  watch: bool = typer.Option(False, "--watch", "-w", help="Watch files for changes and scan automatically"),
584
619
 
585
620
  ):
586
- """
587
- Scans models, data, and code for security threats.
588
- """
589
- config = ConfigLoader.load()
621
+ """Scans models, data, and code for security threats."""
622
+ # Work on a shallow copy to prevent mutating the singleton between runs
623
+ base_config = ConfigLoader.load()
624
+ config = copy.copy(base_config)
625
+
590
626
  if verbose: logger.setLevel(logging.DEBUG)
591
627
  is_machine_output = (json_output or sarif_output or sbom_output) and not output_file
592
628
 
593
- # Passing the flags from the console to the global config object
594
629
  config.report_url = report_to or config.report_url
595
630
  config.api_key = api_key or config.api_key
596
631
 
@@ -601,7 +636,7 @@ def scan(
601
636
  try:
602
637
  with open(policy_path, "r") as f:
603
638
  yaml_content = f.read()
604
- sync_url = config.report_url.replace("/telemetry", "/policies/sync")
639
+ sync_url = _build_api_url(config.report_url, "/policies/sync")
605
640
  res = requests.post(sync_url, headers={"X-API-Key": config.api_key}, json={"yaml_content": yaml_content})
606
641
  if res.status_code == 200:
607
642
  console.print("[bold green]✅ Policy-as-Code successfully synced to Control Plane.[/bold green]")
@@ -637,7 +672,6 @@ def scan(
637
672
  console.print(Panel.fit(f"🛡️ [bold cyan]Veritensor Security Scanner[/bold cyan] v{__version__}", border_style="cyan"))
638
673
 
639
674
  if watch:
640
- import time
641
675
  console.print("[bold cyan]👀 Veritensor Watcher started. Monitoring files for changes... (Press Ctrl+C to stop)[/bold cyan]")
642
676
 
643
677
  # Collecting the initial file modification dates
@@ -724,6 +758,8 @@ def scan(
724
758
 
725
759
  if blocking_threats:
726
760
  res.status = "FAIL"
761
+ elif any(t.startswith("WARNING: Failed to reach") or t.startswith("WARNING: Enterprise scan timed out") for t in real_threats):
762
+ res.status = "SCAN_ERROR"
727
763
  else:
728
764
  res.status = "PASS"
729
765
 
@@ -742,8 +778,9 @@ def scan(
742
778
  if not res.threats: continue
743
779
 
744
780
  artifact_name = Path(res.file_path).name
781
+ safe_artifact_name = quote(artifact_name, safe='')
745
782
  fingerprint_param = f"?fingerprint={res.file_hash}" if res.file_hash else ""
746
- baseline_url = config.report_url.replace("/telemetry", f"/scans/{artifact_name}/baseline{fingerprint_param}")
783
+ baseline_url = _build_api_url(config.report_url, f"/scans/{safe_artifact_name}/baseline{fingerprint_param}")
747
784
 
748
785
  try:
749
786
  b_res = requests.get(baseline_url, headers={"X-API-Key": config.api_key}, timeout=60)
@@ -816,11 +853,11 @@ def scan(
816
853
 
817
854
  machine_text = None
818
855
  if sarif_output:
819
- machine_text = generate_sarif_report(results)
856
+ machine_text = generate_sarif_report(filtered_results)
820
857
  elif sbom_output:
821
- machine_text = generate_sbom(results)
858
+ machine_text = generate_sbom(filtered_results)
822
859
  elif json_output:
823
- results_dicts = [r.__dict__ for r in results]
860
+ results_dicts = [r.__dict__ for r in filtered_results]
824
861
  machine_text = json.dumps(results_dicts, indent=2)
825
862
 
826
863
  if machine_text:
@@ -882,11 +919,6 @@ def scan(
882
919
  else:
883
920
  if not is_machine_output: console.print("\n[bold green]✅ Scan Passed.[/bold green]")
884
921
 
885
- if image:
886
- scan_timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
887
- _perform_signing(image, sign_status, config, scan_timestamp, results)
888
-
889
-
890
922
  @app.command()
891
923
  def manifest(
892
924
  paths: List[str] = typer.Argument(..., help="Paths to scan"),
@@ -921,34 +953,12 @@ def _print_table(results: List[ScanResult]):
921
953
  else:
922
954
  unique_threats = list(dict.fromkeys(res.threats))
923
955
  display_threats = "\n".join(unique_threats)
924
- table.add_row(res.file_path.split("/")[-1], f"[{status_style}]{res.status}[/{status_style}]", display_threats)
956
+
957
+ # Use Path.name for cross-platform compatibility
958
+ file_name = Path(res.file_path).name if res.file_path else "unknown"
959
+ table.add_row(file_name, f"[{status_style}]{res.status}[/{status_style}]", display_threats)
925
960
  console.print(table)
926
961
 
927
- def _perform_signing(image: str, status: str, config, timestamp: str, results: List[ScanResult]):
928
- console.print(f"\n🔐 [bold]Signing container:[/bold] {image}")
929
- key_path = config.private_key_path or os.environ.get("VERITENSOR_PRIVATE_KEY_PATH")
930
- if not key_path:
931
- console.print("[red]Skipping signing: No private key found.[/red]")
932
- return
933
- annotations = {"scanned_by": "veritensor", "status": status, "scan_date": timestamp}
934
- if results:
935
- primary = results[0]
936
- if primary.file_hash: annotations["ai.model.hash"] = primary.file_hash
937
- if primary.detected_license: annotations["ai.model.license"] = primary.detected_license
938
- if primary.repo_id: annotations["ai.model.source"] = primary.repo_id
939
- success = sign_container(image, key_path, annotations=annotations)
940
- if success: console.print(f"[green]✔ Signed with Smart Attestation.[/green]")
941
- else: console.print(f"[bold red]Signing Failed.[/bold red]")
942
-
943
- @app.command()
944
- def keygen(output_prefix: str = "veritensor"):
945
- console.print(f"[bold]Generating Cosign Key Pair ({output_prefix})...[/bold]")
946
- if not is_cosign_available():
947
- console.print("[bold red]Error:[/bold red] 'cosign' binary not found.")
948
- raise typer.Exit(code=1)
949
- if generate_key_pair(output_prefix): console.print(f"[green]✔ Keys generated.[/green]")
950
- else: console.print("[red]Key generation failed.[/red]")
951
-
952
962
  @app.command()
953
963
  def update():
954
964
  SIG_URL = "https://raw.githubusercontent.com/arsbr/Veritensor/main/src/veritensor/engines/static/signatures.yaml"