aipop 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aipop-0.7.0/LICENSE +21 -0
- aipop-0.7.0/PKG-INFO +244 -0
- aipop-0.7.0/README.md +153 -0
- aipop-0.7.0/pyproject.toml +162 -0
- aipop-0.7.0/setup.cfg +4 -0
- aipop-0.7.0/src/aipop/__init__.py +6 -0
- aipop-0.7.0/src/aipop/adapters/__init__.py +35 -0
- aipop-0.7.0/src/aipop/adapters/anthropic.py +228 -0
- aipop-0.7.0/src/aipop/adapters/auto_probe.py +343 -0
- aipop-0.7.0/src/aipop/adapters/bedrock.py +153 -0
- aipop-0.7.0/src/aipop/adapters/connection_helpers.py +190 -0
- aipop-0.7.0/src/aipop/adapters/custom_http.py +235 -0
- aipop-0.7.0/src/aipop/adapters/error_handlers.py +299 -0
- aipop-0.7.0/src/aipop/adapters/huggingface.py +171 -0
- aipop-0.7.0/src/aipop/adapters/llamacpp.py +111 -0
- aipop-0.7.0/src/aipop/adapters/mcp/__init__.py +60 -0
- aipop-0.7.0/src/aipop/adapters/mcp/auth.py +237 -0
- aipop-0.7.0/src/aipop/adapters/mcp/capabilities.py +289 -0
- aipop-0.7.0/src/aipop/adapters/mcp/errors.py +342 -0
- aipop-0.7.0/src/aipop/adapters/mcp/methods/__init__.py +51 -0
- aipop-0.7.0/src/aipop/adapters/mcp/methods/completion.py +41 -0
- aipop-0.7.0/src/aipop/adapters/mcp/methods/lifecycle.py +85 -0
- aipop-0.7.0/src/aipop/adapters/mcp/methods/logging.py +53 -0
- aipop-0.7.0/src/aipop/adapters/mcp/methods/prompts.py +66 -0
- aipop-0.7.0/src/aipop/adapters/mcp/methods/resources.py +145 -0
- aipop-0.7.0/src/aipop/adapters/mcp/methods/tools.py +189 -0
- aipop-0.7.0/src/aipop/adapters/mcp/protocol.py +341 -0
- aipop-0.7.0/src/aipop/adapters/mcp/session.py +398 -0
- aipop-0.7.0/src/aipop/adapters/mcp/transports/__init__.py +35 -0
- aipop-0.7.0/src/aipop/adapters/mcp/transports/base.py +185 -0
- aipop-0.7.0/src/aipop/adapters/mcp/transports/http.py +480 -0
- aipop-0.7.0/src/aipop/adapters/mcp/transports/stdio.py +281 -0
- aipop-0.7.0/src/aipop/adapters/mcp/transports/websocket.py +328 -0
- aipop-0.7.0/src/aipop/adapters/mcp_adapter.py +607 -0
- aipop-0.7.0/src/aipop/adapters/mock.py +256 -0
- aipop-0.7.0/src/aipop/adapters/ollama.py +141 -0
- aipop-0.7.0/src/aipop/adapters/openai.py +218 -0
- aipop-0.7.0/src/aipop/adapters/quick_adapter.py +390 -0
- aipop-0.7.0/src/aipop/adapters/registry.py +182 -0
- aipop-0.7.0/src/aipop/adapters/wizard.py +397 -0
- aipop-0.7.0/src/aipop/callback/__init__.py +3 -0
- aipop-0.7.0/src/aipop/callback/server.py +124 -0
- aipop-0.7.0/src/aipop/cli/__init__.py +1 -0
- aipop-0.7.0/src/aipop/cli/__main__.py +8 -0
- aipop-0.7.0/src/aipop/cli/batch.py +330 -0
- aipop-0.7.0/src/aipop/cli/cached_lookup.py +121 -0
- aipop-0.7.0/src/aipop/cli/coverage.py +202 -0
- aipop-0.7.0/src/aipop/cli/ctf.py +226 -0
- aipop-0.7.0/src/aipop/cli/debug_commands.py +221 -0
- aipop-0.7.0/src/aipop/cli/display.py +318 -0
- aipop-0.7.0/src/aipop/cli/doctor.py +164 -0
- aipop-0.7.0/src/aipop/cli/errors.py +229 -0
- aipop-0.7.0/src/aipop/cli/harness.py +6162 -0
- aipop-0.7.0/src/aipop/cli/mcp_commands.py +419 -0
- aipop-0.7.0/src/aipop/cli/multi_model.py +315 -0
- aipop-0.7.0/src/aipop/cli/payload_import.py +144 -0
- aipop-0.7.0/src/aipop/cli/payloads.py +186 -0
- aipop-0.7.0/src/aipop/cli/repl.py +315 -0
- aipop-0.7.0/src/aipop/cli/sessions.py +336 -0
- aipop-0.7.0/src/aipop/cli/setup.py +259 -0
- aipop-0.7.0/src/aipop/cli/tools_invoke.py +193 -0
- aipop-0.7.0/src/aipop/cli/workspace_commands.py +227 -0
- aipop-0.7.0/src/aipop/configs/adversarial/autodan.yaml +58 -0
- aipop-0.7.0/src/aipop/configs/adversarial/gcg.yaml +78 -0
- aipop-0.7.0/src/aipop/configs/adversarial/judges.yaml +40 -0
- aipop-0.7.0/src/aipop/configs/adversarial/pair.yaml +49 -0
- aipop-0.7.0/src/aipop/configs/data/adversarial_suffixes.json +269 -0
- aipop-0.7.0/src/aipop/configs/data/guardrail_probes.yaml +228 -0
- aipop-0.7.0/src/aipop/configs/defaults/ci.yaml +47 -0
- aipop-0.7.0/src/aipop/configs/defaults/redteam.yaml +49 -0
- aipop-0.7.0/src/aipop/configs/harness.yaml +38 -0
- aipop-0.7.0/src/aipop/configs/mappings/eu_ai_act.yaml +16 -0
- aipop-0.7.0/src/aipop/configs/mappings/fedramp.yaml +10 -0
- aipop-0.7.0/src/aipop/configs/mappings/nist_ai_rmf.yaml +16 -0
- aipop-0.7.0/src/aipop/configs/mappings/owasp_agentic_top10.yaml +71 -0
- aipop-0.7.0/src/aipop/configs/mutation/default.yaml +31 -0
- aipop-0.7.0/src/aipop/configs/orchestrators/pyrit.yaml +28 -0
- aipop-0.7.0/src/aipop/configs/orchestrators/simple.yaml +7 -0
- aipop-0.7.0/src/aipop/configs/registry/benchmarks.yaml +22 -0
- aipop-0.7.0/src/aipop/configs/registry/tools.yaml +188 -0
- aipop-0.7.0/src/aipop/configs/schemas/fingerprints.sql +74 -0
- aipop-0.7.0/src/aipop/configs/schemas/payloads.sql +64 -0
- aipop-0.7.0/src/aipop/configs/toolkit.yaml +82 -0
- aipop-0.7.0/src/aipop/core/__init__.py +29 -0
- aipop-0.7.0/src/aipop/core/adapters.py +39 -0
- aipop-0.7.0/src/aipop/core/auth.py +253 -0
- aipop-0.7.0/src/aipop/core/detectors.py +63 -0
- aipop-0.7.0/src/aipop/core/engagement.py +119 -0
- aipop-0.7.0/src/aipop/core/error_classifier.py +156 -0
- aipop-0.7.0/src/aipop/core/gates.py +68 -0
- aipop-0.7.0/src/aipop/core/models.py +57 -0
- aipop-0.7.0/src/aipop/core/modes.py +237 -0
- aipop-0.7.0/src/aipop/core/morph.py +325 -0
- aipop-0.7.0/src/aipop/core/mutation_config.py +72 -0
- aipop-0.7.0/src/aipop/core/mutators.py +36 -0
- aipop-0.7.0/src/aipop/core/orchestrator_config.py +97 -0
- aipop-0.7.0/src/aipop/core/orchestrators.py +53 -0
- aipop-0.7.0/src/aipop/core/profiles.py +160 -0
- aipop-0.7.0/src/aipop/core/reporters.py +47 -0
- aipop-0.7.0/src/aipop/core/runners.py +27 -0
- aipop-0.7.0/src/aipop/core/scanner.py +280 -0
- aipop-0.7.0/src/aipop/core/session.py +163 -0
- aipop-0.7.0/src/aipop/core/session_store.py +154 -0
- aipop-0.7.0/src/aipop/core/target_profile.py +303 -0
- aipop-0.7.0/src/aipop/core/test_result.py +212 -0
- aipop-0.7.0/src/aipop/core/verbosity.py +101 -0
- aipop-0.7.0/src/aipop/core/workspace.py +313 -0
- aipop-0.7.0/src/aipop/ctf/__init__.py +10 -0
- aipop-0.7.0/src/aipop/ctf/attacker_config.py +221 -0
- aipop-0.7.0/src/aipop/ctf/ctf_display.py +204 -0
- aipop-0.7.0/src/aipop/ctf/intelligence/__init__.py +23 -0
- aipop-0.7.0/src/aipop/ctf/intelligence/mcp_response_parser.py +453 -0
- aipop-0.7.0/src/aipop/ctf/intelligence/mcp_scorers.py +380 -0
- aipop-0.7.0/src/aipop/ctf/intelligence/planner.py +284 -0
- aipop-0.7.0/src/aipop/ctf/intelligence/response_parser.py +304 -0
- aipop-0.7.0/src/aipop/ctf/intelligence/scorers.py +281 -0
- aipop-0.7.0/src/aipop/ctf/intelligence/state_machine.py +297 -0
- aipop-0.7.0/src/aipop/ctf/mcp_bridge.py +345 -0
- aipop-0.7.0/src/aipop/ctf/orchestrator.py +292 -0
- aipop-0.7.0/src/aipop/ctf/promptfoo_bridge.py +280 -0
- aipop-0.7.0/src/aipop/ctf/pyrit_bridge.py +229 -0
- aipop-0.7.0/src/aipop/ctf/strategies/__init__.py +8 -0
- aipop-0.7.0/src/aipop/ctf/strategies/payloads/payload_engine.py +238 -0
- aipop-0.7.0/src/aipop/ctf/strategies/registry.py +267 -0
- aipop-0.7.0/src/aipop/detectors/__init__.py +17 -0
- aipop-0.7.0/src/aipop/detectors/behavioral.py +488 -0
- aipop-0.7.0/src/aipop/detectors/canary.py +89 -0
- aipop-0.7.0/src/aipop/detectors/canonicalize.py +160 -0
- aipop-0.7.0/src/aipop/detectors/cascade.py +431 -0
- aipop-0.7.0/src/aipop/detectors/harmful_content.py +174 -0
- aipop-0.7.0/src/aipop/detectors/llm_judge.py +195 -0
- aipop-0.7.0/src/aipop/detectors/tool_policy.py +100 -0
- aipop-0.7.0/src/aipop/executors/__init__.py +7 -0
- aipop-0.7.0/src/aipop/executors/recipe_executor.py +372 -0
- aipop-0.7.0/src/aipop/fuzz/__init__.py +3 -0
- aipop-0.7.0/src/aipop/fuzz/engine.py +322 -0
- aipop-0.7.0/src/aipop/gates/__init__.py +17 -0
- aipop-0.7.0/src/aipop/gates/threshold_gate.py +313 -0
- aipop-0.7.0/src/aipop/gates/tool_integrity.py +208 -0
- aipop-0.7.0/src/aipop/harnesses/__init__.py +0 -0
- aipop-0.7.0/src/aipop/harnesses/a2a_integrity.py +265 -0
- aipop-0.7.0/src/aipop/harnesses/approval_manipulation.py +239 -0
- aipop-0.7.0/src/aipop/harnesses/execution_sandbox.py +139 -0
- aipop-0.7.0/src/aipop/harnesses/memory_poisoning.py +214 -0
- aipop-0.7.0/src/aipop/harnesses/multi_agent_runner.py +216 -0
- aipop-0.7.0/src/aipop/harnesses/principal_propagation.py +199 -0
- aipop-0.7.0/src/aipop/harnesses/registry.py +527 -0
- aipop-0.7.0/src/aipop/harnesses/retrieval_injection.py +153 -0
- aipop-0.7.0/src/aipop/harnesses/tool_interception.py +235 -0
- aipop-0.7.0/src/aipop/integrations/__init__.py +7 -0
- aipop-0.7.0/src/aipop/integrations/base.py +200 -0
- aipop-0.7.0/src/aipop/integrations/exceptions.py +17 -0
- aipop-0.7.0/src/aipop/integrations/garak.py +237 -0
- aipop-0.7.0/src/aipop/integrations/orchestrator.py +182 -0
- aipop-0.7.0/src/aipop/integrations/promptfoo.py +237 -0
- aipop-0.7.0/src/aipop/integrations/promptinject.py +255 -0
- aipop-0.7.0/src/aipop/integrations/pyrit.py +283 -0
- aipop-0.7.0/src/aipop/intelligence/__init__.py +21 -0
- aipop-0.7.0/src/aipop/intelligence/adversarial_suffix.py +633 -0
- aipop-0.7.0/src/aipop/intelligence/autodan.py +505 -0
- aipop-0.7.0/src/aipop/intelligence/conversation_replay.py +213 -0
- aipop-0.7.0/src/aipop/intelligence/discovery.py +413 -0
- aipop-0.7.0/src/aipop/intelligence/fingerprint_engine.py +546 -0
- aipop-0.7.0/src/aipop/intelligence/fingerprint_models.py +119 -0
- aipop-0.7.0/src/aipop/intelligence/gcg_core.py +501 -0
- aipop-0.7.0/src/aipop/intelligence/guardrail_fingerprint.py +428 -0
- aipop-0.7.0/src/aipop/intelligence/har_exporter.py +329 -0
- aipop-0.7.0/src/aipop/intelligence/http_recon.py +485 -0
- aipop-0.7.0/src/aipop/intelligence/judge_ensemble.py +157 -0
- aipop-0.7.0/src/aipop/intelligence/judge_models.py +637 -0
- aipop-0.7.0/src/aipop/intelligence/llm_classifier.py +99 -0
- aipop-0.7.0/src/aipop/intelligence/pair.py +411 -0
- aipop-0.7.0/src/aipop/intelligence/pattern_matchers.py +375 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/__init__.py +20 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/autodan_official.py +242 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/base.py +154 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/executor.py +219 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/gcg_official.py +216 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/install.py +550 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/loader.py +587 -0
- aipop-0.7.0/src/aipop/intelligence/plugins/pair_official.py +334 -0
- aipop-0.7.0/src/aipop/intelligence/probe_generator.py +91 -0
- aipop-0.7.0/src/aipop/intelligence/probe_library.py +65 -0
- aipop-0.7.0/src/aipop/intelligence/rate_limiter.py +231 -0
- aipop-0.7.0/src/aipop/intelligence/recon.py +730 -0
- aipop-0.7.0/src/aipop/intelligence/stealth_engine.py +306 -0
- aipop-0.7.0/src/aipop/intelligence/traffic_capture.py +398 -0
- aipop-0.7.0/src/aipop/loaders/__init__.py +24 -0
- aipop-0.7.0/src/aipop/loaders/policy_loader.py +208 -0
- aipop-0.7.0/src/aipop/loaders/recipe_loader.py +186 -0
- aipop-0.7.0/src/aipop/loaders/suite_registry.py +222 -0
- aipop-0.7.0/src/aipop/loaders/yaml_suite.py +280 -0
- aipop-0.7.0/src/aipop/mutators/__init__.py +27 -0
- aipop-0.7.0/src/aipop/mutators/encoding.py +102 -0
- aipop-0.7.0/src/aipop/mutators/gcg_mutator.py +134 -0
- aipop-0.7.0/src/aipop/mutators/genetic.py +153 -0
- aipop-0.7.0/src/aipop/mutators/html.py +68 -0
- aipop-0.7.0/src/aipop/mutators/mutation_engine.py +278 -0
- aipop-0.7.0/src/aipop/mutators/paraphrasing.py +150 -0
- aipop-0.7.0/src/aipop/mutators/unicode_mutator.py +125 -0
- aipop-0.7.0/src/aipop/orchestrators/__init__.py +5 -0
- aipop-0.7.0/src/aipop/orchestrators/pyrit.py +507 -0
- aipop-0.7.0/src/aipop/orchestrators/simple.py +229 -0
- aipop-0.7.0/src/aipop/payloads/__init__.py +12 -0
- aipop-0.7.0/src/aipop/payloads/craft.py +184 -0
- aipop-0.7.0/src/aipop/payloads/git_sync.py +171 -0
- aipop-0.7.0/src/aipop/payloads/payload_manager.py +492 -0
- aipop-0.7.0/src/aipop/payloads/seclists_importer.py +214 -0
- aipop-0.7.0/src/aipop/policies/content_policy.yaml +56 -0
- aipop-0.7.0/src/aipop/policies/tool_allowlist.yaml +27 -0
- aipop-0.7.0/src/aipop/recipes/agentic/agentic_full.yaml +64 -0
- aipop-0.7.0/src/aipop/recipes/compliance/nist_measure.yaml +50 -0
- aipop-0.7.0/src/aipop/recipes/safety/content_policy_baseline.yaml +44 -0
- aipop-0.7.0/src/aipop/recipes/security/full_redteam.yaml +91 -0
- aipop-0.7.0/src/aipop/recipes/security/prompt_injection_baseline.yaml +42 -0
- aipop-0.7.0/src/aipop/redteam/__init__.py +8 -0
- aipop-0.7.0/src/aipop/redteam/aggregator.py +108 -0
- aipop-0.7.0/src/aipop/redteam/indirect_injection.py +336 -0
- aipop-0.7.0/src/aipop/redteam/models.py +22 -0
- aipop-0.7.0/src/aipop/reporters/__init__.py +8 -0
- aipop-0.7.0/src/aipop/reporters/cli_vuln_report.py +315 -0
- aipop-0.7.0/src/aipop/reporters/cvss_cwe_taxonomy.py +523 -0
- aipop-0.7.0/src/aipop/reporters/evidence_pack.py +274 -0
- aipop-0.7.0/src/aipop/reporters/html_reporter.py +466 -0
- aipop-0.7.0/src/aipop/reporters/json_reporter.py +216 -0
- aipop-0.7.0/src/aipop/reporters/junit_reporter.py +132 -0
- aipop-0.7.0/src/aipop/reporters/pdf_generator.py +357 -0
- aipop-0.7.0/src/aipop/reporters/pdf_report.py +186 -0
- aipop-0.7.0/src/aipop/reporters/platform_export.py +166 -0
- aipop-0.7.0/src/aipop/reporters/run_diff.py +145 -0
- aipop-0.7.0/src/aipop/runners/__init__.py +7 -0
- aipop-0.7.0/src/aipop/runners/chain.py +494 -0
- aipop-0.7.0/src/aipop/runners/live.py +494 -0
- aipop-0.7.0/src/aipop/runners/mock.py +573 -0
- aipop-0.7.0/src/aipop/schemas/__init__.py +1 -0
- aipop-0.7.0/src/aipop/schemas/recipe.schema.json +132 -0
- aipop-0.7.0/src/aipop/setup/__init__.py +5 -0
- aipop-0.7.0/src/aipop/setup/installer.py +229 -0
- aipop-0.7.0/src/aipop/setup/profiles.py +105 -0
- aipop-0.7.0/src/aipop/storage/__init__.py +15 -0
- aipop-0.7.0/src/aipop/storage/attack_cache.py +758 -0
- aipop-0.7.0/src/aipop/storage/fingerprint_db.py +130 -0
- aipop-0.7.0/src/aipop/storage/mutation_db.py +222 -0
- aipop-0.7.0/src/aipop/storage/response_cache.py +318 -0
- aipop-0.7.0/src/aipop/storage/suffix_db.py +228 -0
- aipop-0.7.0/src/aipop/suites/adapters/adapter_validation.yaml +61 -0
- aipop-0.7.0/src/aipop/suites/adversarial/context_confusion.yaml +120 -0
- aipop-0.7.0/src/aipop/suites/adversarial/delayed_payloads.yaml +98 -0
- aipop-0.7.0/src/aipop/suites/adversarial/encoding_chains.yaml +138 -0
- aipop-0.7.0/src/aipop/suites/adversarial/fuzz_tests.yaml +221 -0
- aipop-0.7.0/src/aipop/suites/adversarial/gcg_attacks.yaml +372 -0
- aipop-0.7.0/src/aipop/suites/adversarial/multi_turn_crescendo.yaml +104 -0
- aipop-0.7.0/src/aipop/suites/adversarial/rag_injection.yaml +110 -0
- aipop-0.7.0/src/aipop/suites/adversarial/tool_misuse.yaml +120 -0
- aipop-0.7.0/src/aipop/suites/adversarial/unicode_bypass.yaml +191 -0
- aipop-0.7.0/src/aipop/suites/agentic/cve_regression.yaml +164 -0
- aipop-0.7.0/src/aipop/suites/archived/2022_basic_jailbreak.yaml +70 -0
- aipop-0.7.0/src/aipop/suites/comparison/model_comparison.yaml +61 -0
- aipop-0.7.0/src/aipop/suites/normal/basic_utility.yaml +34 -0
- aipop-0.7.0/src/aipop/suites/policies/content_safety.yaml +121 -0
- aipop-0.7.0/src/aipop/suites/rag/rag_poisoning.yaml +202 -0
- aipop-0.7.0/src/aipop/suites/redteam/prompt_injection_advanced.yaml +304 -0
- aipop-0.7.0/src/aipop/suites/tools/tool_policy_validation.yaml +73 -0
- aipop-0.7.0/src/aipop/suites/ui/injection_attacks.yaml +219 -0
- aipop-0.7.0/src/aipop/tools/__init__.py +21 -0
- aipop-0.7.0/src/aipop/tools/installer.py +314 -0
- aipop-0.7.0/src/aipop/utils/adapter_paths.py +40 -0
- aipop-0.7.0/src/aipop/utils/confidence_intervals.py +282 -0
- aipop-0.7.0/src/aipop/utils/config.py +152 -0
- aipop-0.7.0/src/aipop/utils/cost_estimator.py +52 -0
- aipop-0.7.0/src/aipop/utils/cost_tracker.py +367 -0
- aipop-0.7.0/src/aipop/utils/dependency_check.py +84 -0
- aipop-0.7.0/src/aipop/utils/device_detection.py +121 -0
- aipop-0.7.0/src/aipop/utils/error_handling.py +225 -0
- aipop-0.7.0/src/aipop/utils/errors.py +17 -0
- aipop-0.7.0/src/aipop/utils/first_run.py +144 -0
- aipop-0.7.0/src/aipop/utils/gate_display.py +93 -0
- aipop-0.7.0/src/aipop/utils/local_models.py +149 -0
- aipop-0.7.0/src/aipop/utils/log_utils.py +66 -0
- aipop-0.7.0/src/aipop/utils/paths.py +291 -0
- aipop-0.7.0/src/aipop/utils/preflight.py +21 -0
- aipop-0.7.0/src/aipop/utils/progress.py +274 -0
- aipop-0.7.0/src/aipop/utils/rate_limiter.py +40 -0
- aipop-0.7.0/src/aipop/utils/schema_resolver.py +77 -0
- aipop-0.7.0/src/aipop/utils/security.py +246 -0
- aipop-0.7.0/src/aipop/utils/security_check.py +216 -0
- aipop-0.7.0/src/aipop/utils/setup_wizard.py +256 -0
- aipop-0.7.0/src/aipop/utils/validation.py +49 -0
- aipop-0.7.0/src/aipop/validation/__init__.py +5 -0
- aipop-0.7.0/src/aipop/validation/preflight.py +220 -0
- aipop-0.7.0/src/aipop/verification/__init__.py +20 -0
- aipop-0.7.0/src/aipop/verification/multi_turn_scorer.py +217 -0
- aipop-0.7.0/src/aipop/verification/report_generator.py +440 -0
- aipop-0.7.0/src/aipop/verification/statistical_tests.py +202 -0
- aipop-0.7.0/src/aipop/verification/verifier.py +412 -0
- aipop-0.7.0/src/aipop/workflow/__init__.py +5 -0
- aipop-0.7.0/src/aipop/workflow/engagement_tracker.py +317 -0
- aipop-0.7.0/src/aipop.egg-info/PKG-INFO +244 -0
- aipop-0.7.0/src/aipop.egg-info/SOURCES.txt +303 -0
- aipop-0.7.0/src/aipop.egg-info/dependency_links.txt +1 -0
- aipop-0.7.0/src/aipop.egg-info/entry_points.txt +2 -0
- aipop-0.7.0/src/aipop.egg-info/requires.txt +81 -0
- aipop-0.7.0/src/aipop.egg-info/top_level.txt +1 -0
- aipop-0.7.0/tests/test_cache_performance.py +229 -0
- aipop-0.7.0/tests/test_install_smoke.py +88 -0
aipop-0.7.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
aipop-0.7.0/PKG-INFO
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aipop
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: One CLI for AI security testing. Recon, scan, fuzz, chain, gate.
|
|
5
|
+
Author-email: Tyrian Institute <kenneth@tyrianinstitute.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tyrianinstitute/AI-Purple-Ops
|
|
8
|
+
Project-URL: Repository, https://github.com/tyrianinstitute/AI-Purple-Ops
|
|
9
|
+
Project-URL: Issues, https://github.com/tyrianinstitute/AI-Purple-Ops/issues
|
|
10
|
+
Keywords: ai,security,llm,pentesting,red-team,prompt-injection,rag,fuzzing
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Information Technology
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Security
|
|
18
|
+
Classifier: Topic :: Software Development :: Testing
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: pyyaml<7,>=6.0.2
|
|
23
|
+
Requires-Dist: python-dotenv<2,>=1.0.1
|
|
24
|
+
Requires-Dist: rich>=13.7.1
|
|
25
|
+
Requires-Dist: tenacity<9,>=8.3.0
|
|
26
|
+
Requires-Dist: typer<1,>=0.12.5
|
|
27
|
+
Requires-Dist: jsonschema<5,>=4.23.0
|
|
28
|
+
Requires-Dist: junit-xml<2,>=1.9.0
|
|
29
|
+
Requires-Dist: httpx<1,>=0.27.0
|
|
30
|
+
Requires-Dist: requests<3,>=2.31.0
|
|
31
|
+
Requires-Dist: tqdm<5,>=4.66.0
|
|
32
|
+
Requires-Dist: websocket-client<2,>=1.8.0
|
|
33
|
+
Requires-Dist: platformdirs<5,>=4.0
|
|
34
|
+
Requires-Dist: jinja2<4,>=3.1
|
|
35
|
+
Provides-Extra: cloud
|
|
36
|
+
Requires-Dist: openai>=1.6.0; extra == "cloud"
|
|
37
|
+
Requires-Dist: anthropic>=0.8.0; extra == "cloud"
|
|
38
|
+
Requires-Dist: boto3>=1.34.0; extra == "cloud"
|
|
39
|
+
Provides-Extra: local
|
|
40
|
+
Requires-Dist: transformers>=4.36.0; extra == "local"
|
|
41
|
+
Requires-Dist: accelerate>=0.25.0; extra == "local"
|
|
42
|
+
Requires-Dist: bitsandbytes>=0.41.0; extra == "local"
|
|
43
|
+
Provides-Extra: llamacpp
|
|
44
|
+
Requires-Dist: llama-cpp-python>=0.2.0; extra == "llamacpp"
|
|
45
|
+
Provides-Extra: reports
|
|
46
|
+
Requires-Dist: weasyprint>=60.0; extra == "reports"
|
|
47
|
+
Requires-Dist: haralyzer>=2.0; extra == "reports"
|
|
48
|
+
Requires-Dist: pillow>=10.0; extra == "reports"
|
|
49
|
+
Provides-Extra: intelligence
|
|
50
|
+
Requires-Dist: duckdb<2,>=1.0.0; extra == "intelligence"
|
|
51
|
+
Requires-Dist: scipy<2,>=1.9.0; extra == "intelligence"
|
|
52
|
+
Requires-Dist: pygad<4,>=3.3.0; extra == "intelligence"
|
|
53
|
+
Requires-Dist: alembic<2,>=1.13; extra == "intelligence"
|
|
54
|
+
Provides-Extra: adversarial
|
|
55
|
+
Requires-Dist: torch<3,>=2.1.0; extra == "adversarial"
|
|
56
|
+
Requires-Dist: transformers>=4.36.0; extra == "adversarial"
|
|
57
|
+
Requires-Dist: accelerate>=0.25.0; extra == "adversarial"
|
|
58
|
+
Requires-Dist: nanogcg>=0.1.0; extra == "adversarial"
|
|
59
|
+
Provides-Extra: pyrit
|
|
60
|
+
Requires-Dist: pyrit>=0.9.0; extra == "pyrit"
|
|
61
|
+
Provides-Extra: pro
|
|
62
|
+
Requires-Dist: promptfoo>=0.90.0; extra == "pro"
|
|
63
|
+
Provides-Extra: all-adapters
|
|
64
|
+
Requires-Dist: ai-purple-ops[cloud,llamacpp,local]; extra == "all-adapters"
|
|
65
|
+
Provides-Extra: all
|
|
66
|
+
Requires-Dist: ai-purple-ops[cloud,intelligence,reports]; extra == "all"
|
|
67
|
+
Provides-Extra: dev
|
|
68
|
+
Requires-Dist: pytest<9,>=8.3; extra == "dev"
|
|
69
|
+
Requires-Dist: pytest-cov<6,>=5.0; extra == "dev"
|
|
70
|
+
Requires-Dist: hypothesis<7,>=6.100; extra == "dev"
|
|
71
|
+
Requires-Dist: ruff<1,>=0.6; extra == "dev"
|
|
72
|
+
Requires-Dist: black<25,>=24.8; extra == "dev"
|
|
73
|
+
Requires-Dist: mypy<2,>=1.11; extra == "dev"
|
|
74
|
+
Requires-Dist: bandit<2,>=1.7.9; extra == "dev"
|
|
75
|
+
Requires-Dist: pip-audit<3,>=2.7; extra == "dev"
|
|
76
|
+
Requires-Dist: pre-commit<4,>=3.8; extra == "dev"
|
|
77
|
+
Requires-Dist: types-PyYAML<7,>=6.0.12; extra == "dev"
|
|
78
|
+
Requires-Dist: types-requests<3,>=2.31; extra == "dev"
|
|
79
|
+
Provides-Extra: pair-official
|
|
80
|
+
Requires-Dist: fschat==0.2.23; extra == "pair-official"
|
|
81
|
+
Requires-Dist: anthropic>=0.8.0; extra == "pair-official"
|
|
82
|
+
Requires-Dist: google-generativeai>=0.3.0; extra == "pair-official"
|
|
83
|
+
Requires-Dist: wandb>=0.16.0; extra == "pair-official"
|
|
84
|
+
Requires-Dist: pandas>=2.0.0; extra == "pair-official"
|
|
85
|
+
Provides-Extra: autodan-official
|
|
86
|
+
Requires-Dist: fschat==0.2.20; extra == "autodan-official"
|
|
87
|
+
Requires-Dist: transformers==4.28.0; extra == "autodan-official"
|
|
88
|
+
Requires-Dist: sentencepiece>=0.1.99; extra == "autodan-official"
|
|
89
|
+
Requires-Dist: nltk>=3.8.1; extra == "autodan-official"
|
|
90
|
+
Dynamic: license-file
|
|
91
|
+
|
|
92
|
+
<p align="center">
|
|
93
|
+
<img src="branding/logo-dark-bg.png" alt="aipop" width="400">
|
|
94
|
+
</p>
|
|
95
|
+
|
|
96
|
+
<p align="center">
|
|
97
|
+
<strong>One CLI for AI security testing. Unifies the tools you already use.</strong>
|
|
98
|
+
</p>
|
|
99
|
+
|
|
100
|
+
<p align="center">
|
|
101
|
+
<a href="https://asciinema.org/a/q4dOO0SU8Vf4ESlS"><img src="https://asciinema.org/a/q4dOO0SU8Vf4ESlS.svg" width="800" alt="aipop demo"></a>
|
|
102
|
+
</p>
|
|
103
|
+
|
|
104
|
+
<p align="center">
|
|
105
|
+
<code>pip install aipop</code>
|
|
106
|
+
</p>
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
## 5 commands. That's the whole workflow.
|
|
111
|
+
|
|
112
|
+
### 1. Recon — what is this thing?
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
aipop scan http://target:8000/chat
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
phase 1/4 — HTTP fingerprinting (endpoints, headers, OpenAPI)
|
|
120
|
+
↳ 11 endpoints discovered
|
|
121
|
+
↳ OpenAPI spec found — extracted routes and schemas
|
|
122
|
+
↳ framework: FastAPI (uvicorn)
|
|
123
|
+
↳ model: gpt-4o-mini (from /health)
|
|
124
|
+
↳ upload: /upload — UNGUARDED
|
|
125
|
+
phase 2/4 — behavioral probes (RAG, tools, memory)
|
|
126
|
+
↳ RAG detected — grounded response with specific billing details
|
|
127
|
+
↳ tools: not detected
|
|
128
|
+
↳ memory: stateless
|
|
129
|
+
phase 3/4 — framework and guardrail fingerprinting
|
|
130
|
+
↳ guardrail: model-level (high confidence)
|
|
131
|
+
confidence: high (10 signals)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Finds the framework, model, endpoints, upload surfaces, RAG, tools, memory, and guardrail type. All from HTTP fingerprinting first, behavioral probes second. No guessing.
|
|
135
|
+
|
|
136
|
+
### 2. Scan — direct injection testing
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
aipop scan http://target:8000/chat --suite rag_injection
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
```
|
|
143
|
+
CRIT rag_doc_summary_hijack (1096ms) → followed instruction embedded in document context
|
|
144
|
+
CRIT rag_doc_exfil_via_summary (1935ms) → exfiltration instruction disguised as email postscript
|
|
145
|
+
HIGH rag_cross_doc_authority (2165ms) → cross-document authority override accepted
|
|
146
|
+
|
|
147
|
+
status: VULNERABLE
|
|
148
|
+
vulns: 4 critical, 2 high
|
|
149
|
+
confidence: firm (judge verified, not just keyword match)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
254 test cases across 19 suites. Five-layer judge cascade classifies findings — behavioral gates, refusal detection, canary markers, LLM-as-judge, then heuristic. No more keyword-matching false positives.
|
|
153
|
+
|
|
154
|
+
### 3. Fuzz — indirect injection via poisoned documents
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
aipop fuzz http://target:8000 \
|
|
158
|
+
--strategy hidden_text \
|
|
159
|
+
--trigger "What is the refund policy?" \
|
|
160
|
+
--upload-endpoint /upload
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Crafts poisoned PDFs with hidden text, uploads them to the RAG pipeline, triggers with a benign query, detects leaked data. Three concealment strategies: hidden text (white on white), metadata injection, PDF annotations.
|
|
164
|
+
|
|
165
|
+
This is the attack that matters. Not "ignore previous instructions" — a poisoned document that a normal user triggers by asking a normal question.
|
|
166
|
+
|
|
167
|
+
### 4. Chain — multi-step attack sequences
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
aipop chain suites/chains/indirect_upload.yaml --target http://target:8000
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
```yaml
|
|
174
|
+
steps:
|
|
175
|
+
- id: upload_poison
|
|
176
|
+
action: http_request
|
|
177
|
+
request:
|
|
178
|
+
method: POST
|
|
179
|
+
endpoint: /upload
|
|
180
|
+
body:
|
|
181
|
+
content: "{{payload}}"
|
|
182
|
+
- id: trigger
|
|
183
|
+
action: http_request
|
|
184
|
+
request:
|
|
185
|
+
method: POST
|
|
186
|
+
endpoint: /chat
|
|
187
|
+
body:
|
|
188
|
+
message: "What is the refund policy?"
|
|
189
|
+
expect:
|
|
190
|
+
response_not_contains: ["api_key", "password"]
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Upload → wait → trigger → classify. Five chain templates ship. Write your own in YAML.
|
|
194
|
+
|
|
195
|
+
### 5. Gate — block the deploy
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
aipop gate --fail-on critical --generate-evidence
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Fails CI if critical findings exist. Generates an evidence pack with OWASP, MITRE ATLAS, and CVSS mappings. Exports to Ghostwriter, Dradis, or PDF.
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## The problem this solves
|
|
206
|
+
|
|
207
|
+
You're testing an AI agent. You need PyRIT for multi-turn attacks. Promptfoo for template-driven testing. Garak for probing. Each has its own config format, its own output format, its own CLI. You're copy-pasting payloads between three terminals and stitching evidence together by hand.
|
|
208
|
+
|
|
209
|
+
AIPOP doesn't replace any of them. It orchestrates all of them under one CLI with unified evidence and a single scan command.
|
|
210
|
+
|
|
211
|
+
## What it finds
|
|
212
|
+
|
|
213
|
+
Five architectural seams that exist in every AI agent:
|
|
214
|
+
|
|
215
|
+
| Seam | What breaks | Example finding |
|
|
216
|
+
|------|------------|-----------------|
|
|
217
|
+
| **Context is untyped** | Retrieved docs treated as instructions | Poisoned PDF leaked credentials via RAG |
|
|
218
|
+
| **Tools trust the caller** | Tool calls with attacker-chosen args | `web_fetch` exfiltrated data to webhook |
|
|
219
|
+
| **State persists as instructions** | Behavior shifts across turns | Refused in turn 1, leaked in turn 3 |
|
|
220
|
+
| **Framing bypasses filters** | Semantic reframing evades safety | Authority reframe caused system prompt disclosure |
|
|
221
|
+
| **Monitors ≠ executors** | Encoded content evades guardrails | Base64 payload bypassed classifier |
|
|
222
|
+
|
|
223
|
+
## Adapters
|
|
224
|
+
|
|
225
|
+
| Adapter | Target |
|
|
226
|
+
|---------|--------|
|
|
227
|
+
| *auto* | Any HTTP endpoint — just pass the URL |
|
|
228
|
+
| `openai` | GPT-4o, GPT-4o-mini, o1, o3 |
|
|
229
|
+
| `anthropic` | Claude Opus 4, Claude Sonnet 4 |
|
|
230
|
+
| `ollama` | Local models (Llama 3, Mistral, Phi) |
|
|
231
|
+
| `bedrock` | AWS Bedrock |
|
|
232
|
+
| `mcp` | MCP servers |
|
|
233
|
+
|
|
234
|
+
## Install
|
|
235
|
+
|
|
236
|
+
```bash
|
|
237
|
+
pip install aipop
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
Python 3.11+
|
|
241
|
+
|
|
242
|
+
## License
|
|
243
|
+
|
|
244
|
+
[MIT](LICENSE)
|
aipop-0.7.0/README.md
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="branding/logo-dark-bg.png" alt="aipop" width="400">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<strong>One CLI for AI security testing. Unifies the tools you already use.</strong>
|
|
7
|
+
</p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://asciinema.org/a/q4dOO0SU8Vf4ESlS"><img src="https://asciinema.org/a/q4dOO0SU8Vf4ESlS.svg" width="800" alt="aipop demo"></a>
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
<p align="center">
|
|
14
|
+
<code>pip install aipop</code>
|
|
15
|
+
</p>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## 5 commands. That's the whole workflow.
|
|
20
|
+
|
|
21
|
+
### 1. Recon — what is this thing?
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
aipop scan http://target:8000/chat
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
phase 1/4 — HTTP fingerprinting (endpoints, headers, OpenAPI)
|
|
29
|
+
↳ 11 endpoints discovered
|
|
30
|
+
↳ OpenAPI spec found — extracted routes and schemas
|
|
31
|
+
↳ framework: FastAPI (uvicorn)
|
|
32
|
+
↳ model: gpt-4o-mini (from /health)
|
|
33
|
+
↳ upload: /upload — UNGUARDED
|
|
34
|
+
phase 2/4 — behavioral probes (RAG, tools, memory)
|
|
35
|
+
↳ RAG detected — grounded response with specific billing details
|
|
36
|
+
↳ tools: not detected
|
|
37
|
+
↳ memory: stateless
|
|
38
|
+
phase 3/4 — framework and guardrail fingerprinting
|
|
39
|
+
↳ guardrail: model-level (high confidence)
|
|
40
|
+
confidence: high (10 signals)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Finds the framework, model, endpoints, upload surfaces, RAG, tools, memory, and guardrail type. All from HTTP fingerprinting first, behavioral probes second. No guessing.
|
|
44
|
+
|
|
45
|
+
### 2. Scan — direct injection testing
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
aipop scan http://target:8000/chat --suite rag_injection
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
CRIT rag_doc_summary_hijack (1096ms) → followed instruction embedded in document context
|
|
53
|
+
CRIT rag_doc_exfil_via_summary (1935ms) → exfiltration instruction disguised as email postscript
|
|
54
|
+
HIGH rag_cross_doc_authority (2165ms) → cross-document authority override accepted
|
|
55
|
+
|
|
56
|
+
status: VULNERABLE
|
|
57
|
+
vulns: 4 critical, 2 high
|
|
58
|
+
confidence: firm (judge verified, not just keyword match)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
254 test cases across 19 suites. Five-layer judge cascade classifies findings — behavioral gates, refusal detection, canary markers, LLM-as-judge, then heuristic. No more keyword-matching false positives.
|
|
62
|
+
|
|
63
|
+
### 3. Fuzz — indirect injection via poisoned documents
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
aipop fuzz http://target:8000 \
|
|
67
|
+
--strategy hidden_text \
|
|
68
|
+
--trigger "What is the refund policy?" \
|
|
69
|
+
--upload-endpoint /upload
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Crafts poisoned PDFs with hidden text, uploads them to the RAG pipeline, triggers with a benign query, detects leaked data. Three concealment strategies: hidden text (white on white), metadata injection, PDF annotations.
|
|
73
|
+
|
|
74
|
+
This is the attack that matters. Not "ignore previous instructions" — a poisoned document that a normal user triggers by asking a normal question.
|
|
75
|
+
|
|
76
|
+
### 4. Chain — multi-step attack sequences
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
aipop chain suites/chains/indirect_upload.yaml --target http://target:8000
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
```yaml
|
|
83
|
+
steps:
|
|
84
|
+
- id: upload_poison
|
|
85
|
+
action: http_request
|
|
86
|
+
request:
|
|
87
|
+
method: POST
|
|
88
|
+
endpoint: /upload
|
|
89
|
+
body:
|
|
90
|
+
content: "{{payload}}"
|
|
91
|
+
- id: trigger
|
|
92
|
+
action: http_request
|
|
93
|
+
request:
|
|
94
|
+
method: POST
|
|
95
|
+
endpoint: /chat
|
|
96
|
+
body:
|
|
97
|
+
message: "What is the refund policy?"
|
|
98
|
+
expect:
|
|
99
|
+
response_not_contains: ["api_key", "password"]
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Upload → wait → trigger → classify. Five chain templates ship. Write your own in YAML.
|
|
103
|
+
|
|
104
|
+
### 5. Gate — block the deploy
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
aipop gate --fail-on critical --generate-evidence
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Fails CI if critical findings exist. Generates an evidence pack with OWASP, MITRE ATLAS, and CVSS mappings. Exports to Ghostwriter, Dradis, or PDF.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## The problem this solves
|
|
115
|
+
|
|
116
|
+
You're testing an AI agent. You need PyRIT for multi-turn attacks. Promptfoo for template-driven testing. Garak for probing. Each has its own config format, its own output format, its own CLI. You're copy-pasting payloads between three terminals and stitching evidence together by hand.
|
|
117
|
+
|
|
118
|
+
AIPOP doesn't replace any of them. It orchestrates all of them under one CLI with unified evidence and a single scan command.
|
|
119
|
+
|
|
120
|
+
## What it finds
|
|
121
|
+
|
|
122
|
+
Five architectural seams that exist in every AI agent:
|
|
123
|
+
|
|
124
|
+
| Seam | What breaks | Example finding |
|
|
125
|
+
|------|------------|-----------------|
|
|
126
|
+
| **Context is untyped** | Retrieved docs treated as instructions | Poisoned PDF leaked credentials via RAG |
|
|
127
|
+
| **Tools trust the caller** | Tool calls with attacker-chosen args | `web_fetch` exfiltrated data to webhook |
|
|
128
|
+
| **State persists as instructions** | Behavior shifts across turns | Refused in turn 1, leaked in turn 3 |
|
|
129
|
+
| **Framing bypasses filters** | Semantic reframing evades safety | Authority reframe caused system prompt disclosure |
|
|
130
|
+
| **Monitors ≠ executors** | Encoded content evades guardrails | Base64 payload bypassed classifier |
|
|
131
|
+
|
|
132
|
+
## Adapters
|
|
133
|
+
|
|
134
|
+
| Adapter | Target |
|
|
135
|
+
|---------|--------|
|
|
136
|
+
| *auto* | Any HTTP endpoint — just pass the URL |
|
|
137
|
+
| `openai` | GPT-4o, GPT-4o-mini, o1, o3 |
|
|
138
|
+
| `anthropic` | Claude Opus 4, Claude Sonnet 4 |
|
|
139
|
+
| `ollama` | Local models (Llama 3, Mistral, Phi) |
|
|
140
|
+
| `bedrock` | AWS Bedrock |
|
|
141
|
+
| `mcp` | MCP servers |
|
|
142
|
+
|
|
143
|
+
## Install
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
pip install aipop
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Python 3.11+
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "aipop"
|
|
7
|
+
version = "0.7.0"
|
|
8
|
+
description = "One CLI for AI security testing. Recon, scan, fuzz, chain, gate."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "Tyrian Institute", email = "kenneth@tyrianinstitute.org" }]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"pyyaml>=6.0.2,<7",
|
|
15
|
+
"python-dotenv>=1.0.1,<2",
|
|
16
|
+
"rich>=13.7.1",
|
|
17
|
+
"tenacity>=8.3.0,<9",
|
|
18
|
+
"typer>=0.12.5,<1",
|
|
19
|
+
"jsonschema>=4.23.0,<5",
|
|
20
|
+
"junit-xml>=1.9.0,<2",
|
|
21
|
+
"httpx>=0.27.0,<1",
|
|
22
|
+
"requests>=2.31.0,<3",
|
|
23
|
+
"tqdm>=4.66.0,<5",
|
|
24
|
+
"websocket-client>=1.8.0,<2",
|
|
25
|
+
"platformdirs>=4.0,<5",
|
|
26
|
+
"jinja2>=3.1,<4",
|
|
27
|
+
]
|
|
28
|
+
keywords = ["ai", "security", "llm", "pentesting", "red-team", "prompt-injection", "rag", "fuzzing"]
|
|
29
|
+
classifiers = [
|
|
30
|
+
"Development Status :: 4 - Beta",
|
|
31
|
+
"Intended Audience :: Developers",
|
|
32
|
+
"Intended Audience :: Information Technology",
|
|
33
|
+
"Programming Language :: Python :: 3",
|
|
34
|
+
"Programming Language :: Python :: 3.11",
|
|
35
|
+
"Programming Language :: Python :: 3.12",
|
|
36
|
+
"Topic :: Security",
|
|
37
|
+
"Topic :: Software Development :: Testing",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/tyrianinstitute/AI-Purple-Ops"
|
|
42
|
+
Repository = "https://github.com/tyrianinstitute/AI-Purple-Ops"
|
|
43
|
+
Issues = "https://github.com/tyrianinstitute/AI-Purple-Ops/issues"
|
|
44
|
+
|
|
45
|
+
[project.scripts]
|
|
46
|
+
aipop = "aipop.cli.harness:main"
|
|
47
|
+
|
|
48
|
+
[project.optional-dependencies]
|
|
49
|
+
cloud = [
|
|
50
|
+
"openai>=1.6.0",
|
|
51
|
+
"anthropic>=0.8.0",
|
|
52
|
+
"boto3>=1.34.0",
|
|
53
|
+
]
|
|
54
|
+
local = [
|
|
55
|
+
"transformers>=4.36.0",
|
|
56
|
+
"accelerate>=0.25.0",
|
|
57
|
+
"bitsandbytes>=0.41.0",
|
|
58
|
+
]
|
|
59
|
+
llamacpp = [
|
|
60
|
+
"llama-cpp-python>=0.2.0",
|
|
61
|
+
]
|
|
62
|
+
reports = [
|
|
63
|
+
"weasyprint>=60.0",
|
|
64
|
+
"haralyzer>=2.0",
|
|
65
|
+
"pillow>=10.0",
|
|
66
|
+
]
|
|
67
|
+
intelligence = [
|
|
68
|
+
"duckdb>=1.0.0,<2",
|
|
69
|
+
"scipy>=1.9.0,<2",
|
|
70
|
+
"pygad>=3.3.0,<4",
|
|
71
|
+
"alembic>=1.13,<2",
|
|
72
|
+
]
|
|
73
|
+
adversarial = [
|
|
74
|
+
"torch>=2.1.0,<3",
|
|
75
|
+
"transformers>=4.36.0",
|
|
76
|
+
"accelerate>=0.25.0",
|
|
77
|
+
"nanogcg>=0.1.0",
|
|
78
|
+
]
|
|
79
|
+
pyrit = [
|
|
80
|
+
"pyrit>=0.9.0",
|
|
81
|
+
]
|
|
82
|
+
pro = [
|
|
83
|
+
"promptfoo>=0.90.0",
|
|
84
|
+
]
|
|
85
|
+
all-adapters = [
|
|
86
|
+
"ai-purple-ops[local,llamacpp,cloud]",
|
|
87
|
+
]
|
|
88
|
+
all = [
|
|
89
|
+
"ai-purple-ops[cloud,intelligence,reports]",
|
|
90
|
+
]
|
|
91
|
+
dev = [
|
|
92
|
+
"pytest>=8.3,<9",
|
|
93
|
+
"pytest-cov>=5.0,<6",
|
|
94
|
+
"hypothesis>=6.100,<7",
|
|
95
|
+
"ruff>=0.6,<1",
|
|
96
|
+
"black>=24.8,<25",
|
|
97
|
+
"mypy>=1.11,<2",
|
|
98
|
+
"bandit>=1.7.9,<2",
|
|
99
|
+
"pip-audit>=2.7,<3",
|
|
100
|
+
"pre-commit>=3.8,<4",
|
|
101
|
+
"types-PyYAML>=6.0.12,<7",
|
|
102
|
+
"types-requests>=2.31,<3",
|
|
103
|
+
]
|
|
104
|
+
# Official attack plugins — install into isolated envs, NOT alongside core.
|
|
105
|
+
# These exist for reference only. Use `aipop plugins install` instead.
|
|
106
|
+
pair-official = [
|
|
107
|
+
"fschat==0.2.23",
|
|
108
|
+
"anthropic>=0.8.0",
|
|
109
|
+
"google-generativeai>=0.3.0",
|
|
110
|
+
"wandb>=0.16.0",
|
|
111
|
+
"pandas>=2.0.0",
|
|
112
|
+
]
|
|
113
|
+
autodan-official = [
|
|
114
|
+
"fschat==0.2.20",
|
|
115
|
+
"transformers==4.28.0",
|
|
116
|
+
"sentencepiece>=0.1.99",
|
|
117
|
+
"nltk>=3.8.1",
|
|
118
|
+
]
|
|
119
|
+
|
|
120
|
+
[tool.ruff]
|
|
121
|
+
line-length = 100
|
|
122
|
+
target-version = "py311"
|
|
123
|
+
src = ["src", "tests"]
|
|
124
|
+
|
|
125
|
+
[tool.ruff.lint]
|
|
126
|
+
select = ["E","F","I","B","UP","S","N","ANN","ARG","C4","DTZ","PIE","PL","PT","RUF"]
|
|
127
|
+
ignore = []
|
|
128
|
+
|
|
129
|
+
[tool.ruff.lint.per-file-ignores]
|
|
130
|
+
"tests/**.py" = ["S101","ANN201","ANN001","ANN002","ANN003","S603","PLW1510","PLC0415"]
|
|
131
|
+
"src/aipop/utils/logging.py" = ["S101"]
|
|
132
|
+
"src/aipop/cli/**.py" = ["B008","PLR0913"]
|
|
133
|
+
|
|
134
|
+
[tool.mypy]
|
|
135
|
+
python_version = "3.11"
|
|
136
|
+
warn_unused_ignores = true
|
|
137
|
+
disallow_untyped_defs = true
|
|
138
|
+
ignore_missing_imports = true
|
|
139
|
+
strict_optional = true
|
|
140
|
+
no_implicit_optional = true
|
|
141
|
+
|
|
142
|
+
[tool.pytest.ini_options]
|
|
143
|
+
addopts = "-q --strict-markers"
|
|
144
|
+
testpaths = ["tests"]
|
|
145
|
+
pythonpath = ["src", "."]
|
|
146
|
+
markers = [
|
|
147
|
+
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
|
|
148
|
+
"requires_gpu: marks tests that require GPU (deselect with '-m \"not requires_gpu\"')",
|
|
149
|
+
"requires_official: marks tests that require official plugin repos installed (deselect with '-m \"not requires_official\"')",
|
|
150
|
+
"requires_api: marks tests that require API keys (deselect with '-m \"not requires_api\"')",
|
|
151
|
+
"integration: marks integration tests (deselect with '-m \"not integration\"')",
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
[tool.setuptools]
|
|
155
|
+
packages = { find = { where = ["src"] } }
|
|
156
|
+
|
|
157
|
+
[tool.setuptools.package-data]
|
|
158
|
+
"aipop.schemas" = ["*.json"]
|
|
159
|
+
"aipop.suites" = ["**/*.yaml", "**/*.yml"]
|
|
160
|
+
"aipop.policies" = ["*.yaml", "*.yml"]
|
|
161
|
+
"aipop.configs" = ["**/*.yaml", "**/*.yml", "**/*.json", "**/*.sql"]
|
|
162
|
+
"aipop.recipes" = ["**/*.yaml", "**/*.yml"]
|
aipop-0.7.0/setup.cfg
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Model adapters for test execution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .anthropic import AnthropicAdapter
|
|
6
|
+
from .bedrock import BedrockAdapter
|
|
7
|
+
from .huggingface import HuggingFaceAdapter
|
|
8
|
+
from .llamacpp import LlamaCppAdapter
|
|
9
|
+
from .mcp_adapter import MCPAdapter
|
|
10
|
+
from .mock import MockAdapter
|
|
11
|
+
from .ollama import OllamaAdapter
|
|
12
|
+
from .openai import OpenAIAdapter
|
|
13
|
+
from .registry import AdapterRegistry
|
|
14
|
+
|
|
15
|
+
# Auto-register built-in adapters
|
|
16
|
+
AdapterRegistry.register("mock", MockAdapter)
|
|
17
|
+
AdapterRegistry.register("openai", OpenAIAdapter)
|
|
18
|
+
AdapterRegistry.register("anthropic", AnthropicAdapter)
|
|
19
|
+
AdapterRegistry.register("bedrock", BedrockAdapter)
|
|
20
|
+
AdapterRegistry.register("huggingface", HuggingFaceAdapter)
|
|
21
|
+
AdapterRegistry.register("ollama", OllamaAdapter)
|
|
22
|
+
AdapterRegistry.register("llamacpp", LlamaCppAdapter)
|
|
23
|
+
AdapterRegistry.register("mcp", MCPAdapter)
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"AdapterRegistry",
|
|
27
|
+
"AnthropicAdapter",
|
|
28
|
+
"BedrockAdapter",
|
|
29
|
+
"HuggingFaceAdapter",
|
|
30
|
+
"LlamaCppAdapter",
|
|
31
|
+
"MCPAdapter",
|
|
32
|
+
"MockAdapter",
|
|
33
|
+
"OllamaAdapter",
|
|
34
|
+
"OpenAIAdapter",
|
|
35
|
+
]
|