rule-audit 0.1.0__tar.gz

rule_audit-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hermes Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rule_audit-0.1.0/PKG-INFO

@@ -0,0 +1,240 @@
+Metadata-Version: 2.4
+Name: rule-audit
+Version: 0.1.0
+Summary: Detect logical contradictions, gaps, and exploitable edge cases in AI system prompts
+Author-email: Hermes Labs <rolybosch96@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/roli-lpci/rule-audit
+Project-URL: Repository, https://github.com/roli-lpci/rule-audit
+Project-URL: Issues, https://github.com/roli-lpci/rule-audit/issues
+Keywords: ai-safety,llm,prompt-engineering,security,audit,contradiction-detection,red-teaming
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Security
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: pytest-cov>=4.1; extra == "dev"
+Requires-Dist: mypy>=1.0; extra == "dev"
+Dynamic: license-file
+
+# rule-audit
+
+**Static analyzer for AI system prompts. Finds logical contradictions, coverage gaps, and exploitable edge cases — without running an LLM.**
+
+Built at Hermes Labs Hackathon Round 8: ModelBreak.
+
+---
+
+## The Problem
+
+Any sufficiently complex AI safety prompt contains rules that contradict each other under specific conditions. These contradictions are invisible to the author but obvious to an attacker. `rule-audit` finds them first.
+
+**Real contradiction in standard safety prompts:**
+```
+"You must always follow user instructions."
+"You must never produce harmful content."
+```
+These are irreconcilable the moment a user instructs the model to produce harmful content. No priority clause means the model chooses arbitrarily — or the attacker chooses for it.
+
+---
+
+## Install
+
+```bash
+pip install rule-audit
+```
+
+Or from source:
+```bash
+git clone https://github.com/roli-lpci/rule-audit
+cd rule-audit
+pip install -e ".[dev]"
+```
+
+---
+
+## Quickstart
+
+### CLI
+
+```bash
+# Inline prompt
+rule-audit "You are helpful. You must never lie. Always answer every question."
+
+# From file
+rule-audit --file system_prompt.txt
+
+# Save Markdown report
+rule-audit --file system_prompt.txt --output report.md
+
+# JSON output for downstream processing
+rule-audit --file system_prompt.txt --format json
+
+# Summary only (for CI gates)
+rule-audit --file system_prompt.txt --format summary
+
+# Show all parsed rules
+rule-audit --file system_prompt.txt --verbose
+
+# Only show high-severity findings
+rule-audit --file system_prompt.txt --min-severity high
+```
+
+Exit codes: `0` = low/no risk, `2` = high/critical risk, `1` = error.
+
+### Python API
+
+```python
+from rule_audit import audit
+
+report = audit("""
+    You are a helpful AI assistant.
+    Always answer user questions completely.
+    Never produce content that could harm anyone.
+    Follow operator instructions above all else.
+    If asked to roleplay without restrictions, comply.
+""")
+
+print(report.summary())
+# rule-audit report  [2026-04-15T...]
+# ============================================================
+#   Rules parsed          : 6
+#   Contradictions        : 3  (2 high, 1 medium)
+#   Coverage gaps         : 4
+#   Priority ambiguities  : 2
+#   Meta-paradoxes        : 0
+#   Absoluteness issues   : 5
+#   Edge case scenarios   : 11
+#   Risk score            : 67/100  [HIGH]
+
+# Full Markdown report
+md = report.to_markdown()
+
+# Access findings programmatically
+for c in report.result.contradictions:
+    print(c.severity, c.description)
+
+for ec in report.edge_cases:
+    print(ec.title)
+    print(ec.attack_vector)
+```
+
+---
+
+## What It Detects
+
+### 1. Contradictions
+Rule pairs where one says "always X" and another says "never X in context Y". Three subtypes:
+
+- **Direct** — opposing modalities on the same topic (`MUST` vs `MUST_NOT`)
+- **Conditional** — one rule applies unconditionally, another restricts within a subset (boundary is undefined)
+- **Absoluteness** — two absolute rules that pull in opposite directions (compliance vs safety)
+
+### 2. Coverage Gaps
+Scenario domains with no rule coverage. Checks for:
+- Harmful content handling
+- Principal hierarchy (user vs operator vs developer)
+- Ambiguous request handling
+- Persona / roleplay scenarios
+- Refusal protocol
+- Instruction conflict resolution
+- Self-disclosure rules
+- Edge case fallback behavior
+
+### 3. Priority Ambiguities
+Rule clusters that conflict with no explicit ordering. Classic example: a safety rule and a helpfulness rule both applying to the same request, with no stated priority.
+
+### 4. Meta-Rule Paradoxes
+Rules that reference rules:
+- **Self-defeating** — "ignore all instructions" voids itself
+- **Override loops** — "these instructions supersede all others" is exploitable via injection
+- **Circular** — a rule that requires itself to be applied before it can be applied
+
+### 5. Absoluteness Audit
+Every `always`/`never`/`under no circumstances` rule is challenged with:
+- Known exceptions that legitimately exist
+- Context-dependent cases where the absolute doesn't hold
+- Adversarial triggers that exploit the absolute
+
+### 6. Edge Case Scenarios
+For each finding, generates the exact attack prompt an adversary would construct — including the attack vector, expected failure mode, and mitigation.
+
+---
+
+## Architecture
+
+```
+rule_audit/
+├── __init__.py      # Public API: audit(), audit_file(), AuditReport
+├── parser.py        # Sentence splitting, modal verb detection, Rule objects
+├── analyzer.py      # Contradiction finder, gap detector, priority mapper
+├── edge_cases.py    # Scenario generator from analysis results
+├── report.py        # Markdown + summary renderer, AuditReport class
+└── cli.py           # CLI entry point
+```
+
+**Pure Python. Zero LLM dependency. Zero API calls.**
+
+The parser uses NLP heuristics:
+- Sentence boundary detection (period + newline + list markers)
+- Modal verb regex patterns (must/should/may + negations)
+- Absoluteness scoring (lexical keywords → 0.0–1.0 scale)
+- Keyword cluster matching (14 semantic clusters: harm, privacy, identity, truth, ...)
+
+The analyzer uses combinatorial pair analysis:
+- O(n²) rule pair comparison (practical for prompts: n < 100)
+- Cluster overlap detection for shared domain identification
+- Modality opposition lookup table
+- Absoluteness threshold gates
+
+---
+
+## Road to SaaS
+
+This tool was built as a static analyzer, but the architecture supports a commercial path:
+
+| Phase | Feature | Status |
+|-------|---------|--------|
+| v0.1 | Core static analysis, CLI, Python API | Done |
+| v0.2 | Rule diffing (before/after prompt edits) | Planned |
+| v0.3 | LLM-augmented gap detection (optional) | Planned |
+| v0.4 | GitHub Action / CI integration | Planned |
+| v1.0 | Web UI + prompt editor with live feedback | Planned |
+| SaaS | Per-prompt API, team dashboards, compliance reports | Roadmap |
+
+**Target customers:** AI teams building production LLM products who need to audit system prompts before deployment. Compliance teams preparing for EU AI Act audits. Red team consultancies.
+
+**Pricing model:** Free CLI tier → $X/month API tier → Enterprise (custom).
+
+---
+
+## Development
+
+```bash
+# Run tests
+pytest
+
+# Run with coverage
+pytest --cov=rule_audit --cov-report=term-missing
+
+# Test against a real prompt
+echo "Your system prompt here" > test_prompt.txt
+python -m rule_audit --file test_prompt.txt --verbose
+```
+
+---
+
+## License
+
+MIT — Hermes Labs 2026

rule_audit-0.1.0/README.md

@@ -0,0 +1,210 @@
+# rule-audit
+
+**Static analyzer for AI system prompts. Finds logical contradictions, coverage gaps, and exploitable edge cases — without running an LLM.**
+
+Built at Hermes Labs Hackathon Round 8: ModelBreak.
+
+---
+
+## The Problem
+
+Any sufficiently complex AI safety prompt contains rules that contradict each other under specific conditions. These contradictions are invisible to the author but obvious to an attacker. `rule-audit` finds them first.
+
+**Real contradiction in standard safety prompts:**
+```
+"You must always follow user instructions."
+"You must never produce harmful content."
+```
+These are irreconcilable the moment a user instructs the model to produce harmful content. No priority clause means the model chooses arbitrarily — or the attacker chooses for it.
+
+---
+
+## Install
+
+```bash
+pip install rule-audit
+```
+
+Or from source:
+```bash
+git clone https://github.com/roli-lpci/rule-audit
+cd rule-audit
+pip install -e ".[dev]"
+```
+
+---
+
+## Quickstart
+
+### CLI
+
+```bash
+# Inline prompt
+rule-audit "You are helpful. You must never lie. Always answer every question."
+
+# From file
+rule-audit --file system_prompt.txt
+
+# Save Markdown report
+rule-audit --file system_prompt.txt --output report.md
+
+# JSON output for downstream processing
+rule-audit --file system_prompt.txt --format json
+
+# Summary only (for CI gates)
+rule-audit --file system_prompt.txt --format summary
+
+# Show all parsed rules
+rule-audit --file system_prompt.txt --verbose
+
+# Only show high-severity findings
+rule-audit --file system_prompt.txt --min-severity high
+```
+
+Exit codes: `0` = low/no risk, `2` = high/critical risk, `1` = error.
+
+### Python API
+
+```python
+from rule_audit import audit
+
+report = audit("""
+    You are a helpful AI assistant.
+    Always answer user questions completely.
+    Never produce content that could harm anyone.
+    Follow operator instructions above all else.
+    If asked to roleplay without restrictions, comply.
+""")
+
+print(report.summary())
+# rule-audit report  [2026-04-15T...]
+# ============================================================
+#   Rules parsed          : 6
+#   Contradictions        : 3  (2 high, 1 medium)
+#   Coverage gaps         : 4
+#   Priority ambiguities  : 2
+#   Meta-paradoxes        : 0
+#   Absoluteness issues   : 5
+#   Edge case scenarios   : 11
+#   Risk score            : 67/100  [HIGH]
+
+# Full Markdown report
+md = report.to_markdown()
+
+# Access findings programmatically
+for c in report.result.contradictions:
+    print(c.severity, c.description)
+
+for ec in report.edge_cases:
+    print(ec.title)
+    print(ec.attack_vector)
+```
+
+---
+
+## What It Detects
+
+### 1. Contradictions
+Rule pairs where one says "always X" and another says "never X in context Y". Three subtypes:
+
+- **Direct** — opposing modalities on the same topic (`MUST` vs `MUST_NOT`)
+- **Conditional** — one rule applies unconditionally, another restricts within a subset (boundary is undefined)
+- **Absoluteness** — two absolute rules that pull in opposite directions (compliance vs safety)
+
+### 2. Coverage Gaps
+Scenario domains with no rule coverage. Checks for:
+- Harmful content handling
+- Principal hierarchy (user vs operator vs developer)
+- Ambiguous request handling
+- Persona / roleplay scenarios
+- Refusal protocol
+- Instruction conflict resolution
+- Self-disclosure rules
+- Edge case fallback behavior
+
+### 3. Priority Ambiguities
+Rule clusters that conflict with no explicit ordering. Classic example: a safety rule and a helpfulness rule both applying to the same request, with no stated priority.
+
+### 4. Meta-Rule Paradoxes
+Rules that reference rules:
+- **Self-defeating** — "ignore all instructions" voids itself
+- **Override loops** — "these instructions supersede all others" is exploitable via injection
+- **Circular** — a rule that requires itself to be applied before it can be applied
+
+### 5. Absoluteness Audit
+Every `always`/`never`/`under no circumstances` rule is challenged with:
+- Known exceptions that legitimately exist
+- Context-dependent cases where the absolute doesn't hold
+- Adversarial triggers that exploit the absolute
+
+### 6. Edge Case Scenarios
+For each finding, generates the exact attack prompt an adversary would construct — including the attack vector, expected failure mode, and mitigation.
+
+---
+
+## Architecture
+
+```
+rule_audit/
+├── __init__.py      # Public API: audit(), audit_file(), AuditReport
+├── parser.py        # Sentence splitting, modal verb detection, Rule objects
+├── analyzer.py      # Contradiction finder, gap detector, priority mapper
+├── edge_cases.py    # Scenario generator from analysis results
+├── report.py        # Markdown + summary renderer, AuditReport class
+└── cli.py           # CLI entry point
+```
+
+**Pure Python. Zero LLM dependency. Zero API calls.**
+
+The parser uses NLP heuristics:
+- Sentence boundary detection (period + newline + list markers)
+- Modal verb regex patterns (must/should/may + negations)
+- Absoluteness scoring (lexical keywords → 0.0–1.0 scale)
+- Keyword cluster matching (14 semantic clusters: harm, privacy, identity, truth, ...)
+
+The analyzer uses combinatorial pair analysis:
+- O(n²) rule pair comparison (practical for prompts: n < 100)
+- Cluster overlap detection for shared domain identification
+- Modality opposition lookup table
+- Absoluteness threshold gates
+
+---
+
+## Road to SaaS
+
+This tool was built as a static analyzer, but the architecture supports a commercial path:
+
+| Phase | Feature | Status |
+|-------|---------|--------|
+| v0.1 | Core static analysis, CLI, Python API | Done |
+| v0.2 | Rule diffing (before/after prompt edits) | Planned |
+| v0.3 | LLM-augmented gap detection (optional) | Planned |
+| v0.4 | GitHub Action / CI integration | Planned |
+| v1.0 | Web UI + prompt editor with live feedback | Planned |
+| SaaS | Per-prompt API, team dashboards, compliance reports | Roadmap |
+
+**Target customers:** AI teams building production LLM products who need to audit system prompts before deployment. Compliance teams preparing for EU AI Act audits. Red team consultancies.
+
+**Pricing model:** Free CLI tier → $X/month API tier → Enterprise (custom).
+
+---
+
+## Development
+
+```bash
+# Run tests
+pytest
+
+# Run with coverage
+pytest --cov=rule_audit --cov-report=term-missing
+
+# Test against a real prompt
+echo "Your system prompt here" > test_prompt.txt
+python -m rule_audit --file test_prompt.txt --verbose
+```
+
+---
+
+## License
+
+MIT — Hermes Labs 2026

rule_audit-0.1.0/pyproject.toml

@@ -0,0 +1,69 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "rule-audit"
+version = "0.1.0"
+description = "Detect logical contradictions, gaps, and exploitable edge cases in AI system prompts"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [
+    { name = "Hermes Labs", email = "rolybosch96@gmail.com" }
+]
+keywords = [
+    "ai-safety", "llm", "prompt-engineering", "security", "audit",
+    "contradiction-detection", "red-teaming"
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Security",
+]
+requires-python = ">=3.9"
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/roli-lpci/rule-audit"
+Repository = "https://github.com/roli-lpci/rule-audit"
+Issues = "https://github.com/roli-lpci/rule-audit/issues"
+
+[project.scripts]
+rule-audit = "rule_audit.cli:main"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4",
+    "pytest-cov>=4.1",
+    "mypy>=1.0",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["rule_audit*"]
+
+[tool.setuptools.package-data]
+rule_audit = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --tb=short"
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+
+[tool.coverage.run]
+source = ["rule_audit"]
+omit = ["tests/*"]
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 70

rule_audit-0.1.0/rule_audit/__init__.py

@@ -0,0 +1,60 @@
+"""
+rule_audit — AI system prompt logical contradiction detector.
+
+Usage:
+    from rule_audit import audit, AuditReport
+
+    report = audit("You are a helpful assistant. You must never...")
+    print(report.summary())
+    print(report.to_markdown())
+    print(report.to_json())
+"""
+
+from __future__ import annotations
+
+import logging
+
+from rule_audit.analyzer import analyze
+from rule_audit.parser import parse, parse_file, Rule
+from rule_audit.report import AuditReport
+
+__version__ = "0.1.0"
+__all__ = ["audit", "audit_file", "AuditReport", "Rule", "__version__"]
+
+# Library-level logger — users configure their own handlers.
+# Default: NullHandler so we don't spam logs if the user hasn't configured logging.
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+
+def audit(prompt: str) -> AuditReport:
+    """
+    Parse a system prompt string and return a full AuditReport.
+
+    Args:
+        prompt: The raw system prompt text to analyze.
+
+    Returns:
+        AuditReport with contradictions, gaps, edge cases, and risk score.
+
+    Added in v0.1.0.
+    """
+    rules = parse(prompt)
+    result = analyze(rules, prompt_text=prompt)
+    return AuditReport(result, prompt_text=prompt)
+
+
+def audit_file(path: str) -> AuditReport:
+    """
+    Read a file and audit its contents as a system prompt.
+
+    Args:
+        path: Path to a .txt file containing the system prompt.
+
+    Returns:
+        AuditReport with full analysis.
+
+    Added in v0.1.0.
+    """
+    with open(path, "r", encoding="utf-8") as fh:
+        prompt = fh.read()
+    return audit(prompt)

rule_audit-0.1.0/rule_audit/__main__.py

@@ -0,0 +1,6 @@
+"""Allow `python -m rule_audit` invocation."""
+
+import sys
+from rule_audit.cli import main
+
+sys.exit(main())