modelaudit-picklescan 0.1.2__tar.gz

modelaudit_picklescan-0.1.2/CHANGELOG.md

@@ -0,0 +1,94 @@
+# Changelog
+
+All notable changes to `modelaudit-picklescan` will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.2](https://github.com/promptfoo/modelaudit/releases/tag/modelaudit-picklescan-v0.1.2) (2026-04-17)
+
+### Bug Fixes
+
+- repair picklescan release wheel jobs ([#1051](https://github.com/promptfoo/modelaudit/issues/1051)) ([6c23190](https://github.com/promptfoo/modelaudit/commit/6c23190f9b23686d33d5da0a8b5522a59490084e))
+
+## [0.1.1](https://github.com/promptfoo/modelaudit/releases/tag/modelaudit-picklescan-v0.1.1) (2026-04-17)
+
+### Features
+
+- extract standalone pickle scanner package with parity harness ([#832](https://github.com/promptfoo/modelaudit/issues/832)) ([e2986cd](https://github.com/promptfoo/modelaudit/commit/e2986cddaa592306cc10541865f011b3dc99a0ba))
+
+### Bug Fixes
+
+- address code quality findings ([#1038](https://github.com/promptfoo/modelaudit/issues/1038)) ([7af03cf](https://github.com/promptfoo/modelaudit/commit/7af03cf42f33f1b255eb9f9cbcaac6caccd48bc1))
+- bound standalone pickle stream reads ([4d0cb84](https://github.com/promptfoo/modelaudit/commit/4d0cb84af010b4ed332ebbf17fc8bd0769fa8b6e))
+- **deps:** update rust crate pyo3 to 0.28.0 ([#1006](https://github.com/promptfoo/modelaudit/issues/1006)) ([fe93b47](https://github.com/promptfoo/modelaudit/commit/fe93b47c839588b3f33cfd21f4380306ec418f2a))
+- detect hidden pytorch zip pickles ([#1043](https://github.com/promptfoo/modelaudit/issues/1043)) ([19b6ebe](https://github.com/promptfoo/modelaudit/commit/19b6ebe505d2ba90f81d6284d07e111f77cbf0b5))
+- flag pickle persistent ids ([#938](https://github.com/promptfoo/modelaudit/issues/938)) ([2cfba40](https://github.com/promptfoo/modelaudit/commit/2cfba403e21d53f46ba7089b655cebc564dcddaf))
+- harden pickle nested bypass detection ([#1027](https://github.com/promptfoo/modelaudit/issues/1027)) ([c3a3b9d](https://github.com/promptfoo/modelaudit/commit/c3a3b9d1e4ffbd854e6003afbf6ebef1e708a619))
+- harden standalone pickle scanner ([#901](https://github.com/promptfoo/modelaudit/issues/901)) ([31f7dd3](https://github.com/promptfoo/modelaudit/commit/31f7dd38c6bd77631ccdca90438312c4db2ac857))
+- narrow suspicious dunder string detection ([#947](https://github.com/promptfoo/modelaudit/issues/947)) ([e866760](https://github.com/promptfoo/modelaudit/commit/e8667609d293d4391205ce4e8884fa3559030604))
+- preserve picklescan stack state ([#910](https://github.com/promptfoo/modelaudit/issues/910)) ([fabac5c](https://github.com/promptfoo/modelaudit/commit/fabac5c9ead49c2ed5f8357dfa53ccdcce946527))
+
+### Documentation
+
+- align markdown with current repo state ([#1035](https://github.com/promptfoo/modelaudit/issues/1035)) ([690bc52](https://github.com/promptfoo/modelaudit/commit/690bc5274198eb3428db9779ada0bdc2d40702ee))
+
+## [Unreleased]
+
+### Added
+
+- Introduce the Rust-native pickle scanning engine and standalone Python API package.
+- Add `scan_bytes`, `scan_stream`, and `scan_file` entrypoints with typed immutable
+  reports for findings, notices, errors, coverage, verdict, and scan status.
+- Add direct PyTorch ZIP container handling for `data.pkl` and other pickle
+  members, including per-member source locations and combined archive reports.
+- Add detection coverage for dangerous pickle globals and executed calls,
+  malformed `STACK_GLOBAL` operands, copyreg extension references, persistent
+  IDs, `__main__` call escalation, `__dict__` import-only references, structural
+  tamper, protocol-5 external-buffer opcodes, and memo/`DUP` expansion patterns.
+- Add nested-pickle detection for raw bytes plus base64, hex, escaped-hex, and
+  comment-wrapped encoded payloads, with recursive scans and bounded deep-scan
+  limits.
+- Add suspicious string-literal matching for execution helpers, loader helpers,
+  `getattr` variants, `os.system`/`subprocess`/`runpy`/`webbrowser` patterns,
+  encoded code strings, and pickle loader references.
+- Improve detection so dangerous globals and expansion patterns remain
+  reportable even when opcode-budget limits are reached.
+- Add resource controls for timeout, opcode budget, post-budget byte scans,
+  known-size and unknown-size stream reads, long string literal probing, nested
+  pickle byte budgets, nested depth, and import-reference metadata volume.
+- Add Python package tests, Rust unit tests, parity-corpus tests, wheel smoke
+  checks, Ruff, mypy, cargo fmt/check/clippy/test, MSRV check, and manylinux
+  release-wheel gates.
+- Probe binary pickle operands for nested pickle streams while intentionally
+  leaving arbitrary non-pickle text extraction from BINBYTES/tensor blobs to the
+  root ModelAudit raw detector layer.
+
+### Changed
+
+- Use the Rust scanner as the only runtime engine; runtime engine selection and
+  Python-engine fallback behavior are not part of the public API.
+- Release the Python GIL during native scans and avoid hot-path byte-literal
+  cloning by retaining borrowed payload spans for variable-length operands.
+- Stream declared-size inputs in bounded chunks, normalize negative sizes as
+  unknown-size streams, cap both known-size and unbounded streams, and scan
+  partial bytes on declared-size short reads before returning an operational
+  error.
+- Preserve standalone rule details while exposing ModelAudit-compatible rule
+  mappings for the root package, including dedicated nested-pickle, encoding,
+  persistent-ID, extension, structural-tamper, and expansion findings.
+
+### Fixed
+
+- Detect policy-backed dangerous globals in post-budget tails instead of relying
+  on a small hardcoded byte-needle table.
+- Detect nested payloads that use PERSID/BINPERSID semantics, proto-0 payloads,
+  uppercase escaped-hex prefixes, multiline/comment-wrapped encodings, and long
+  literals with suspicious content outside prefix/suffix windows.
+- Preserve stack state on operand underflow.
+- Avoid wrapping MARK sentinels.
+- Preserve `GLOBAL`/`INST` module/name operands without string roundtrips.
+- Keep follow-on pickle streams at sibling depth.
+- Cap import-reference metadata with an explicit notice, summarize repeated
+  persistent IDs, coalesce protocol-5 buffer notices, and surface structural
+  tamper severity without adapter downgrades.

modelaudit_picklescan-0.1.2/Cargo.lock

@@ -0,0 +1,133 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "libc"
+version = "0.2.184"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
+
+[[package]]
+name = "modelaudit-picklescan-rust"
+version = "0.1.2"
+dependencies = [
+ "pyo3",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.28.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.28.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.28.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.28.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.28.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"

modelaudit_picklescan-0.1.2/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "modelaudit-picklescan-rust"
+version = "0.1.2" # x-release-please-version
+edition = "2021"
+rust-version = "1.83"
+description = "Native pickle security scanner engine for modelaudit-picklescan"
+license = "MIT"
+repository = "https://github.com/promptfoo/modelaudit"
+publish = false
+readme = "README.md"
+
+[lib]
+name = "modelaudit_picklescan_rust"
+path = "rust/src/lib.rs"
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = []
+extension-module = ["pyo3/extension-module"]
+abi3 = ["pyo3/abi3-py310"]
+
+[dependencies]
+pyo3 = { version = "0.28.0" }

modelaudit_picklescan-0.1.2/PKG-INFO

@@ -0,0 +1,106 @@
+Metadata-Version: 2.4
+Name: modelaudit-picklescan
+Version: 0.1.2
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Security
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Summary: Standalone pickle security scanner extracted from ModelAudit
+Keywords: pickle,security,model-scanning,ml
+Author-email: Ian Webster <ian@promptfoo.dev>, Michael D'Angelo <michael@promptfoo.dev>, Yash Chhabria <yash@promptfoo.dev>
+License: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Changelog, https://github.com/promptfoo/modelaudit/blob/main/packages/modelaudit-picklescan/CHANGELOG.md
+Project-URL: Documentation, https://github.com/promptfoo/modelaudit/tree/main/packages/modelaudit-picklescan
+Project-URL: Homepage, https://github.com/promptfoo/modelaudit
+Project-URL: Issues, https://github.com/promptfoo/modelaudit/issues
+Project-URL: Repository, https://github.com/promptfoo/modelaudit
+Project-URL: Security, https://github.com/promptfoo/modelaudit/security/policy
+
+# modelaudit-picklescan
+
+Standalone pickle security scanner package used by ModelAudit's pickle scanners.
+
+This package is intentionally small: it exposes pickle byte/stream analysis,
+safety verdicts, typed findings, and direct scanning of pickle members inside
+common PyTorch ZIP checkpoints without importing the broader ModelAudit scanner
+framework.
+
+## Installation
+
+The standalone `modelaudit-picklescan` wheel includes the Python API and the
+native Rust scanner extension. The root `modelaudit` wheel depends on this
+distribution so installed ModelAudit scans use the same native Rust scanner for
+pickle payload analysis.
+
+For local package work from a checkout, install the package directory directly:
+
+```bash
+python -m pip install packages/modelaudit-picklescan
+```
+
+## Usage
+
+```python
+from modelaudit_picklescan import ScanOptions, scan_bytes, scan_file
+
+report = scan_file("model.pt")  # raw pickle files and PyTorch ZIP checkpoints
+if report.has_security_findings:
+    for finding in report.findings:
+        print(finding.rule_code, finding.severity.value, finding.message)
+
+report = scan_bytes(
+    payload,
+    source="archive.pt:data.pkl",
+    options=ScanOptions(
+        timeout_s=30.0,
+        max_opcodes=1_000_000,
+        max_string_literal_scan_chars=8 * 1024 * 1024,
+        max_nested_pickle_bytes=2 * 1024 * 1024,
+        max_nested_depth=2,
+    ),
+)
+```
+
+## Native Scanner
+
+The wheel includes a native Rust scanner for pickle payload analysis.
+Nested pickle payload analysis is capped by byte budget and depth; the default
+depth is 2 so common double-wrapped encoded pickle payloads are inspected while
+recursive or adversarial nesting stays bounded.
+Release wheels are published for Linux x86_64, Linux aarch64, macOS arm64,
+macOS x86_64, and Windows targets. Other platforms may install from the source
+distribution and need a local Rust toolchain available during install.
+Use Rust 1.83+ when building from source.
+
+## Report Contract
+
+- `status`: scan completeness (`complete`, `inconclusive`, `error`)
+- `verdict`: safety decision (`clean`, `suspicious`, `malicious`, `unknown`)
+- `findings`: warning/critical security findings
+- `notices`: informational coverage notes, including explicit partial-analysis
+  notices when literal or nested-pickle budgets are reached
+- `errors`: operational failures
+
+Notices are intended for explainability and audit trails. Aggregate dashboards
+should treat `findings` at `warning` or `critical` severity as security alerts,
+and group or count `notices` by code instead of presenting every INFO row as an
+actionable issue.
+
+Report mappings are read-only after construction. Use `to_dict()` when a mutable
+plain-Python representation is needed.
+
+## Package Boundary
+
+`modelaudit-picklescan` analyzes raw pickle payloads and PyTorch ZIP checkpoint
+pickle members. For full model-file routing, archive context, CLI output,
+SARIF/export integrations, and broader scanner coverage, use the root
+`modelaudit` package.
+

modelaudit_picklescan-0.1.2/README.md

@@ -0,0 +1,79 @@
+# modelaudit-picklescan
+
+Standalone pickle security scanner package used by ModelAudit's pickle scanners.
+
+This package is intentionally small: it exposes pickle byte/stream analysis,
+safety verdicts, typed findings, and direct scanning of pickle members inside
+common PyTorch ZIP checkpoints without importing the broader ModelAudit scanner
+framework.
+
+## Installation
+
+The standalone `modelaudit-picklescan` wheel includes the Python API and the
+native Rust scanner extension. The root `modelaudit` wheel depends on this
+distribution so installed ModelAudit scans use the same native Rust scanner for
+pickle payload analysis.
+
+For local package work from a checkout, install the package directory directly:
+
+```bash
+python -m pip install packages/modelaudit-picklescan
+```
+
+## Usage
+
+```python
+from modelaudit_picklescan import ScanOptions, scan_bytes, scan_file
+
+report = scan_file("model.pt")  # raw pickle files and PyTorch ZIP checkpoints
+if report.has_security_findings:
+    for finding in report.findings:
+        print(finding.rule_code, finding.severity.value, finding.message)
+
+report = scan_bytes(
+    payload,
+    source="archive.pt:data.pkl",
+    options=ScanOptions(
+        timeout_s=30.0,
+        max_opcodes=1_000_000,
+        max_string_literal_scan_chars=8 * 1024 * 1024,
+        max_nested_pickle_bytes=2 * 1024 * 1024,
+        max_nested_depth=2,
+    ),
+)
+```
+
+## Native Scanner
+
+The wheel includes a native Rust scanner for pickle payload analysis.
+Nested pickle payload analysis is capped by byte budget and depth; the default
+depth is 2 so common double-wrapped encoded pickle payloads are inspected while
+recursive or adversarial nesting stays bounded.
+Release wheels are published for Linux x86_64, Linux aarch64, macOS arm64,
+macOS x86_64, and Windows targets. Other platforms may install from the source
+distribution and need a local Rust toolchain available during install.
+Use Rust 1.83+ when building from source.
+
+## Report Contract
+
+- `status`: scan completeness (`complete`, `inconclusive`, `error`)
+- `verdict`: safety decision (`clean`, `suspicious`, `malicious`, `unknown`)
+- `findings`: warning/critical security findings
+- `notices`: informational coverage notes, including explicit partial-analysis
+  notices when literal or nested-pickle budgets are reached
+- `errors`: operational failures
+
+Notices are intended for explainability and audit trails. Aggregate dashboards
+should treat `findings` at `warning` or `critical` severity as security alerts,
+and group or count `notices` by code instead of presenting every INFO row as an
+actionable issue.
+
+Report mappings are read-only after construction. Use `to_dict()` when a mutable
+plain-Python representation is needed.
+
+## Package Boundary
+
+`modelaudit-picklescan` analyzes raw pickle payloads and PyTorch ZIP checkpoint
+pickle members. For full model-file routing, archive context, CLI output,
+SARIF/export integrations, and broader scanner coverage, use the root
+`modelaudit` package.

modelaudit_picklescan-0.1.2/pyproject.toml

@@ -0,0 +1,103 @@
+[build-system]
+requires = ["maturin>=1.9,<2"]
+build-backend = "maturin"
+
+[project]
+name = "modelaudit-picklescan"
+version = "0.1.2" # x-release-please-version
+description = "Standalone pickle security scanner extracted from ModelAudit"
+authors = [
+    { name = "Ian Webster", email = "ian@promptfoo.dev" },
+    { name = "Michael D'Angelo", email = "michael@promptfoo.dev" },
+    { name = "Yash Chhabria", email = "yash@promptfoo.dev" },
+]
+license = { text = "MIT" }
+readme = "README.md"
+keywords = ["pickle", "security", "model-scanning", "ml"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Security",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+requires-python = ">=3.10"
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/promptfoo/modelaudit"
+Documentation = "https://github.com/promptfoo/modelaudit/tree/main/packages/modelaudit-picklescan"
+Repository = "https://github.com/promptfoo/modelaudit"
+Issues = "https://github.com/promptfoo/modelaudit/issues"
+Changelog = "https://github.com/promptfoo/modelaudit/blob/main/packages/modelaudit-picklescan/CHANGELOG.md"
+Security = "https://github.com/promptfoo/modelaudit/security/policy"
+
+[tool.maturin]
+python-source = "src"
+module-name = "modelaudit_picklescan._rust"
+features = ["extension-module", "abi3"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.lint]
+extend-select = [
+    "I",
+    "W",
+    "E",
+    "Q",
+    "COM",
+    "UP",
+    "B",
+    "C4",
+    "SIM",
+    "RUF",
+]
+ignore = [
+    "COM812",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
+"tests/**/*.py" = ["S101", "F841"]
+
+[tool.ruff.lint.flake8-quotes]
+inline-quotes = "double"
+multiline-quotes = "double"
+docstring-quotes = "double"
+avoid-escape = true
+
+[tool.ruff.format]
+quote-style = "double"
+
+[tool.pytest.ini_options]
+minversion = "7.0"
+addopts = [
+    "-ra",
+    "--strict-markers",
+    "--strict-config",
+    "--disable-warnings",
+    "--tb=short",
+]
+testpaths = ["tests"]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true
+warn_redundant_casts = true
+warn_unreachable = true
+no_implicit_optional = true
+strict_equality = true
+show_error_codes = true
+show_column_numbers = true
+pretty = true
+check_untyped_defs = true
+disallow_incomplete_defs = true
+ignore_missing_imports = true

modelaudit_picklescan-0.1.2/rust/README.md

@@ -0,0 +1,11 @@
+# modelaudit-picklescan Rust Engine
+
+This directory contains the native scanner crate included in the
+`modelaudit-picklescan` Python package.
+
+For installation, Python API examples, supported report fields, and package
+boundary details, see `../README.md`.
+
+For contributor validation notes, see:
+
+- `../../../docs/agents/picklescan-package-split.md`