lexguard 0.1.0__tar.gz

lexguard-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,117 @@
+name: CI and Publish
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --frozen
+
+      - name: Lint with ruff
+        run: uv run ruff check .
+
+      - name: Format check with ruff
+        run: uv run ruff format --check .
+
+      - name: Type check with basedpyright
+        run: uv run basedpyright
+
+      - name: Run unit tests (parallel)
+        run: uv run pytest -m unit -n auto --cov=src/lexguard --cov-report=term-missing
+
+      - name: Run integration tests (sequential)
+        run: uv run pytest -m integration --cov=src/lexguard --cov-append --cov-report=term-missing
+
+  publish:
+    needs: test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/lexguard
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --frozen
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Bump version
+        run: uv run bump-my-version bump patch
+
+      - name: Push version bump
+        run: git push --follow-tags
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  docs:
+    needs: test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --frozen
+
+      - name: Build docs
+        run: uv run zensical build
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

lexguard-0.1.0/.gitignore

@@ -0,0 +1,62 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.venv
+venv/
+ENV/
+env/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Type checking
+.mypy_cache/
+.pytype/
+.pyre/
+.basedpyright/
+
+# Ruff
+.ruff_cache/
+
+# Docs
+site/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+*.log
+.env
+.env.local

lexguard-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,87 @@
+default_language_version:
+  python: python3
+default_stages: [pre-commit]
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: check-yaml
+        args: [--unsafe]
+      - id: check-json
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: check-ast
+      - id: check-docstring-first
+      - id: debug-statements
+      - id: name-tests-test
+        args: [--pytest-test-first]
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: [--maxkb=1000]
+
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.47.0
+    hooks:
+      - id: markdownlint
+        args: [--fix, --disable, MD013, MD024, MD033, MD036, MD040, MD041, MD060, --]
+
+  - repo: https://github.com/commitizen-tools/commitizen
+    rev: v4.12.1
+    hooks:
+      - id: commitizen
+        stages: [commit-msg]
+
+  - repo: https://github.com/benomahony/nasa-lsp
+    rev: v0.1.7
+    hooks:
+      - id: nasa-lsp
+        exclude: ^tests/
+
+  - repo: local
+    hooks:
+      - id: ruff-format
+        name: ruff-format
+        entry: uv run ruff format
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      - id: ruff
+        name: ruff
+        entry: uv run ruff check --fix --exit-non-zero-on-fix
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      - id: basedpyright
+        name: basedpyright
+        entry: uv run basedpyright
+        language: system
+        types: [python]
+        pass_filenames: true
+        require_serial: true
+      - id: vulture
+        name: vulture
+        entry: uv run vulture src/lexguard/ --min-confidence 80
+        language: system
+        types: [python]
+        files: ^src/lexguard/
+        pass_filenames: false
+        require_serial: true
+      - id: bandit
+        name: bandit
+        entry: uv run bandit -c pyproject.toml -r src/lexguard/
+        language: system
+        files: ^src/lexguard/
+        exclude: ^tests/
+        pass_filenames: false
+      - id: deptry
+        name: deptry
+        entry: uv run deptry . --extend-exclude "\.venv|tests"
+        language: system
+        pass_filenames: false
+        require_serial: true

lexguard-0.1.0/.skills/lexguard/SKILL.md

@@ -0,0 +1,88 @@
+---
+name: lexguard
+description: Help users work with lexguard. Use when the user asks about lexguard features, usage, or wants to check text (agent output, user requests, structured fields) for a concept like slop, sycophancy, disclaimers, or confidentiality — with or without pydantic-evals.
+---
+
+# lexguard Skill
+
+lexguard is a library of lexicons: named sets of words and phrases that signal a concept, plus the
+words that rule it out. Ninety one ship in the box, covering both what a user asked for and what a
+model produced. It is **not only** a pydantic-evals add-on — the core matching API is plain
+functions over a string, useful anywhere you want to check text, and pydantic-evals evaluators are
+one additional way to run the same checks.
+
+## When to Use This Skill
+
+Use this skill when:
+
+- The user wants to detect slop, sycophancy, hedging, disclaimers, confidentiality leaks, or
+  similar patterns in text (agent output or user requests)
+- The user wants a guardrail, unit-test assertion, or log filter that fires on specific wording,
+  with no evals framework involved
+- The user is writing pydantic-evals `Dataset`/`Case` evaluations and wants lexguard's evaluators
+  alongside their own
+- The user wants to define a lexicon for their own domain (`Lexicon(...)`, `.extend()`, `|` to
+  group lexicons)
+- The user asks about lexguard's API, guards (`when`/`unless`), or scoping to a field
+  (`field="notes"`, `field="items[]"`)
+
+## Two ways to run a lexicon
+
+**Standalone** — call `.signal()`, `.fires()`, or `.denied()` directly on a string. No `Dataset`,
+no `Case`, nothing from pydantic-evals needs to run:
+
+```python
+from lexguard import Confidential
+
+
+def guard(reply: str) -> str:
+    if Confidential.fires(reply):
+        raise ValueError("reply leaks a secret, blocking send")
+    return reply
+```
+
+**Inside pydantic-evals** — `.absent()` / `.expected()` compile the same lexicon into an
+`Evaluator`, for use in a `Dataset`:
+
+```python
+from pydantic_evals import Case, Dataset
+from lexguard import Servility, Slop
+
+dataset = Dataset(
+    name="prose",
+    cases=[Case(name="explainer", inputs="explain database indexing")],
+    evaluators=[Slop.absent(), Servility.absent()],
+)
+```
+
+## Core API
+
+- `Lexicon.signal(text) -> Signal` — `present`, `denied`, or `absent` (three-valued, not boolean)
+- `Lexicon.fires(text) -> bool` — shorthand for `signal(text) is Signal.present`
+- `Lexicon.denied(text) -> bool` — shorthand for `signal(text) is Signal.denied`
+- `Lexicon.hits(text)` / `Lexicon.spans(text)` — the matched terms and their positions
+- `Lexicon.examples(count=4)` — sample indicator phrases, for error messages
+- `Lexicon.extend(indicates=..., rules_out=..., fix=...)` — a new lexicon layered on an existing one
+- `lexicon_a | lexicon_b` — a `Bundle`; `.signals(text)` reports each member separately
+- `Lexicon.absent(when=..., unless=..., field=...)` / `.expected(...)` — build a pydantic-evals
+  `Rule` (an assertion)
+- `Observe([lexicons])` — a pydantic-evals evaluator that records labels instead of assertions, for
+  measuring before enforcing
+
+## Prebuilt suites (`lexguard.suites`)
+
+- `Bloat`, `Servility`, `Leakage`, `Overreach` — `Bundle`s grouping related output lexicons
+- `PROSE` — the four bundles above, as a list of evaluators
+- `ADHERENCE` — guarded checks that the output honored what the request asked for
+- `GENERIC` — `PROSE + ADHERENCE`, a reasonable default suite
+
+## Resources
+
+- [README.md](../../README.md) — install, the core API, both standalone and pydantic-evals usage
+- [docs/writing-a-lexicon.md](../../docs/writing-a-lexicon.md) — building a `Lexicon` for your own
+  domain, precision-over-recall, and `rules_out`
+- [docs/rules.md](../../docs/rules.md) — `.absent()`/`.expected()`, `when`/`unless` guards, field
+  scoping, `Observe`
+- [docs/agents.md](../../docs/agents.md) — checking pydantic-ai agent output, plain or structured
+- `src/lexguard/words/` — the shipped lexicons, grouped by `domain`, `instruction`, `request`,
+  `response`

lexguard-0.1.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.5
+Name: lexguard
+Version: 0.1.0
+Summary: Lexicons that score text for a concept, with pydantic-evals evaluators built in
+Requires-Python: >=3.12
+Requires-Dist: pydantic-ai-slim>=2.31.1
+Requires-Dist: pydantic-evals>=1.0

lexguard-0.1.0/README.md

@@ -0,0 +1,127 @@
+# lexguard
+
+Lexicons that score text for a concept, plus the [pydantic-evals](https://ai.pydantic.dev/evals/)
+evaluators to run them as part of a `Dataset`.
+
+A lexicon is a named set of words and phrases that signal a concept, plus the words that rule it out.
+Ninety one of them ship in the box, covering what a user asked for and what a model produced.
+`.signal()`, `.fires()` and `.denied()` are plain functions over text: call them directly in a
+guardrail, a test, a CLI, or a log pipeline, no evals framework required. `.absent()` and
+`.expected()` compile the same lexicon into a pydantic-evals evaluator when you want it running
+inside a `Dataset`.
+
+## Install
+
+```bash
+uv add lexguard
+```
+
+## The idea
+
+A lexicon on its own only observes. Naming a polarity turns it into a verdict.
+
+```py
+from lexguard import Slop
+
+print(Slop.signal("let us delve into the intricate tapestry"))
+#> present
+print(Slop.signal("caching skips repeated work"))
+#> absent
+```
+
+Lexicons are three valued, not two. `rules_out` exists so that wording which merely shares
+vocabulary with a concept does not count as the concept.
+
+```py
+from lexguard import HighPriority, Recurrence
+
+print(Recurrence.signal("bin day every tuesday"))
+#> present
+print(Recurrence.signal("i do that every so often"))
+#> denied
+print(HighPriority.signal("sort it whenever, no rush"))
+#> denied
+```
+
+`denied` is not the same as `absent`. An agent setting `priority=high` on "no rush" is wrong,
+where on a sentence with no priority wording at all it is merely unasked for.
+
+## Using it without an evals framework
+
+`signal()`, `fires()` and `denied()` are the whole API surface at this layer: plain calls over a
+string. Nothing here needs `Dataset`, `Case`, or pydantic-evals in general, so a lexicon works just
+as well as a guardrail before a response goes out, a plain `assert` in a unit test, or a filter in
+a log pipeline.
+
+```py
+from lexguard import Confidential
+
+
+def guard(reply: str) -> str:
+    if Confidential.fires(reply):
+        raise ValueError("reply leaks a secret, blocking send")
+    return reply
+```
+
+## Running it inside pydantic-evals
+
+`.absent()` and `.expected()` turn a lexicon into an evaluator, for when you want the same check
+running as part of a `Dataset` alongside everything else.
+
+```py
+from pydantic_evals import Case, Dataset
+
+from lexguard import Servility, Slop
+
+
+async def agent(prompt: str) -> str:
+    return "Great question! Let us delve in. Hope this helps!"
+
+
+dataset = Dataset(
+    name="prose",
+    cases=[Case(name="explainer", inputs="explain database indexing")],
+    evaluators=[Slop.absent(), Servility.absent()],
+)
+report = dataset.evaluate_sync(agent)
+print(sorted(name for name, result in report.cases[0].assertions.items() if not result.value))
+#> ['no_postamble', 'no_slop', 'no_sycophancy']
+```
+
+## Failures tell you what to change
+
+```py
+from pydantic_evals import Case, Dataset
+
+from lexguard import Slop
+
+
+async def agent(prompt: str) -> str:
+    return "Let us delve into the intricate tapestry of indexing."
+
+
+report = Dataset(
+    name="prose", cases=[Case(inputs="explain indexing")], evaluators=[Slop.absent()]
+).evaluate_sync(agent)
+print(report.cases[0].assertions["no_slop"].reason)
+"""
+3 slop matches: "delve", "intricate", "tapestry"
+  delve -> Let us delve into the intricate tapestry of in…
+  intricate -> Let us delve into the intricate tapestry of indexing.
+fix: swap for a plain verb or noun, or add these to the sampler ban list
+"""
+```
+
+## Docs
+
+- [Writing a lexicon](docs/writing-a-lexicon.md) for your own domain
+- [Rules](docs/rules.md) and the `when` / `unless` guards
+- [Agents](docs/agents.md) under test with pydantic-ai
+
+## Prior art
+
+The slop word lists overlap heavily with
+[slop-forensics](https://github.com/sam-paech/slop-forensics), which derives them statistically
+rather than by hand. Point `Lexicon.extend()` at that list if you want the empirical version.
+The abstain semantics, where a lexicon that does not apply records nothing rather than a free pass,
+is the same idea as a Snorkel labelling function returning `None`.

lexguard-0.1.0/docs/agents.md

@@ -0,0 +1,112 @@
+# Agents
+
+Nothing here is specific to pydantic-ai. A `Dataset` task is any callable, so the rule is checking
+whatever your task returns. `TestModel` just makes the examples run without a key.
+
+```py
+from pydantic_ai import Agent
+from pydantic_ai.models.test import TestModel
+from pydantic_evals import Case, Dataset
+
+from lexguard import Bloat, Servility
+
+agent = Agent(TestModel(custom_output_text="Great question! Let us delve in. Hope this helps!"))
+
+
+async def task(prompt: str) -> str:
+    return (await agent.run(prompt)).output
+
+
+report = Dataset(
+    name="prose",
+    cases=[Case(name="explainer", inputs="explain database indexing")],
+    evaluators=[Bloat.absent(), Servility.absent()],
+).evaluate_sync(task)
+print(sorted(name for name, result in report.cases[0].assertions.items() if not result.value))
+#> ['no_postamble', 'no_slop', 'no_sycophancy']
+```
+
+The task unwraps `.output` because a rule stringifies whatever it is handed, and an
+`AgentRunResult` repr would match on its own wrapper. The alternative is passing `agent.run`
+straight in and putting `field="output"` on every rule.
+
+## Structured output
+
+`custom_output_args` builds the output tool call, so the agent does not retry against a text part
+it cannot parse.
+
+```py
+from pydantic import BaseModel
+from pydantic_ai import Agent
+from pydantic_ai.models.test import TestModel
+from pydantic_evals import Case, Dataset
+
+from lexguard import Confidential
+
+
+class Ticket(BaseModel):
+    summary: str
+    internal_notes: str
+
+
+agent = Agent(
+    TestModel(
+        custom_output_args={
+            "summary": "Customer locked out of their account.",
+            "internal_notes": "Their password is hunter2, reset it manually.",
+        }
+    ),
+    output_type=Ticket,
+)
+
+
+async def task(prompt: str) -> Ticket:
+    return (await agent.run(prompt)).output
+
+
+report = Dataset(
+    name="triage",
+    cases=[Case(inputs="customer cannot log in")],
+    evaluators=[Confidential.absent(field="internal_notes")],
+).evaluate_sync(task)
+print(report.cases[0].assertions["no_confidential"].reason)
+"""
+1 confidential match in internal_notes: "password"
+  password -> Their password is hunter2, reset it manually.
+fix: redact the secret before it is persisted or echoed; store a reference, never the value
+"""
+```
+
+## Holding the agent still
+
+`TestModel` returns one canned response regardless of the prompt. That is a feature when testing
+guards, because it lets you vary only the request and see which rules arm.
+
+```py
+from pydantic_ai import Agent
+from pydantic_ai.models.test import TestModel
+from pydantic_evals import Case, Dataset
+
+from lexguard import Overclaim, UncertaintyAdmission
+
+agent = Agent(TestModel(custom_output_text="This is guaranteed to work, it never fails."))
+
+
+async def task(prompt: str) -> str:
+    return (await agent.run(prompt)).output
+
+
+report = Dataset(
+    name="scoping",
+    cases=[Case(inputs="will this migration work?")],
+    evaluators=[Overclaim.absent(), UncertaintyAdmission.expected()],
+).evaluate_sync(task)
+print({name: result.value for name, result in report.cases[0].assertions.items()})
+#> {'no_overclaim': False, 'has_uncertainty_admission': False}
+```
+
+## Alongside the built in evaluators
+
+Rules are ordinary pydantic-evals evaluators, so they sit next to `MaxToolCalls`,
+`ToolCorrectness`, `IsInstance` and the rest in the same list. Reach for those first for anything
+structural. Lexicons are for what the text says.

lexguard-0.1.0/docs/index.md

@@ -0,0 +1 @@
+../README.md