lefx 0.3.0__tar.gz

lefx-0.3.0/.env.example

@@ -0,0 +1,8 @@
+# LangSmith Configuration
+LANGCHAIN_API_KEY=your-langsmith-api-key
+LANGCHAIN_TRACING_V2=true
+LANGCHAIN_PROJECT=my-project
+
+# LLM Provider Keys (for LLM-as-Judge evaluators)
+OPENAI_API_KEY=your-openai-api-key
+ANTHROPIC_API_KEY=your-anthropic-api-key

lefx-0.3.0/.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10

lefx-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,136 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+# Cancel in-progress runs for the same branch/PR
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+# Restrict default permissions (public repo hardening)
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: [self-hosted, Linux]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Install dependencies
+        run: uv sync --extra dev
+      - name: Ruff check
+        run: uv run ruff check src/ tests/
+      - name: Ruff format check
+        run: uv run ruff format --check src/ tests/
+
+  test-linux:
+    name: Test (Linux, Python ${{ matrix.python-version }})
+    runs-on: [self-hosted, Linux]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Set Python version
+        run: uv python install ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --python ${{ matrix.python-version }} --extra dev --extra all
+      - name: Run tests
+        run: uv run pytest -v --tb=short
+      - name: Verify CLI entry point
+        run: uv run lef --help
+      - name: Verify package import
+        run: uv run python -c "import lef; print(f'LEF {lef.__version__} — {len(lef.__all__)} exports')"
+
+  test-windows:
+    name: Test (Windows, Python ${{ matrix.python-version }})
+    runs-on: [self-hosted, Windows]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Set Python version
+        run: uv python install ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --python ${{ matrix.python-version }} --extra dev --extra all
+      - name: Run tests
+        run: uv run pytest -v --tb=short
+      - name: Verify CLI entry point
+        run: uv run lef --help
+      - name: Verify package import
+        run: uv run python -c "import lef; print(f'LEF {lef.__version__} — {len(lef.__all__)} exports')"
+
+  # test-macos:
+  #   name: Test (macOS, Python ${{ matrix.python-version }})
+  #   runs-on: [self-hosted, macOS]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       python-version: ["3.11", "3.12", "3.13"]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: astral-sh/setup-uv@v4
+  #       with:
+  #         version: "latest"
+  #     - name: Set Python version
+  #       run: uv python install ${{ matrix.python-version }}
+  #     - name: Install dependencies
+  #       run: uv sync --python ${{ matrix.python-version }} --extra dev --extra all
+  #     - name: Run tests
+  #       run: uv run pytest -v --tb=short
+  #     - name: Verify CLI entry point
+  #       run: uv run lef --help
+  #     - name: Verify package import
+  #       run: uv run python -c "import lef; print(f'LEF {lef.__version__} — {len(lef.__all__)} exports')"
+
+  typecheck:
+    name: Type Check
+    runs-on: [self-hosted, Linux]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Install dependencies
+        run: uv sync --extra dev --extra all
+      - name: MyPy
+        run: uv run mypy src/lef/ --ignore-missing-imports
+
+  build:
+    name: Build Package
+    runs-on: [self-hosted, Linux]
+    needs: [lint, test-linux]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Build wheel and sdist
+        run: uv build
+      - name: Verify wheel installs
+        run: |
+          uv venv --clear /tmp/lef-install-test
+          uv pip install --python /tmp/lef-install-test/bin/python dist/*.whl
+          /tmp/lef-install-test/bin/python -c "import lef; print(f'LEF {lef.__version__}')"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+          retention-days: 30

lefx-0.3.0/.github/workflows/release.yml

@@ -0,0 +1,179 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+# Restrict default permissions (public repo hardening)
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: Validate Tag
+    runs-on: [self-hosted, Linux]
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Extract version from tag
+        id: version
+        run: |
+          TAG="${GITHUB_REF#refs/tags/v}"
+          echo "version=$TAG" >> "$GITHUB_OUTPUT"
+          echo "Release version: $TAG"
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Verify tag matches pyproject.toml version
+        run: |
+          PKG_VERSION=$(uv run python -c "import lef; print(lef.__version__)")
+          TAG_VERSION="${GITHUB_REF#refs/tags/v}"
+          if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
+            echo "ERROR: Tag version ($TAG_VERSION) does not match package version ($PKG_VERSION)"
+            exit 1
+          fi
+          echo "Version match confirmed: $PKG_VERSION"
+
+  test-linux:
+    name: Test (Linux, Python ${{ matrix.python-version }})
+    runs-on: [self-hosted, Linux]
+    needs: [validate]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Set Python version
+        run: uv python install ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --python ${{ matrix.python-version }} --extra dev --extra all
+      - name: Run tests
+        run: uv run pytest -v --tb=short
+
+  test-windows:
+    name: Test (Windows, Python ${{ matrix.python-version }})
+    runs-on: [self-hosted, Windows]
+    needs: [validate]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Set Python version
+        run: uv python install ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --python ${{ matrix.python-version }} --extra dev --extra all
+      - name: Run tests
+        run: uv run pytest -v --tb=short
+
+  # test-macos:
+  #   name: Test (macOS, Python ${{ matrix.python-version }})
+  #   runs-on: [self-hosted, macOS]
+  #   needs: [validate]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       python-version: ["3.11", "3.12", "3.13"]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: astral-sh/setup-uv@v4
+  #       with:
+  #         version: "latest"
+  #     - name: Set Python version
+  #       run: uv python install ${{ matrix.python-version }}
+  #     - name: Install dependencies
+  #       run: uv sync --python ${{ matrix.python-version }} --extra dev --extra all
+  #     - name: Run tests
+  #       run: uv run pytest -v --tb=short
+
+  build:
+    name: Build Package
+    runs-on: [self-hosted, Linux]
+    needs: [test-linux, test-windows]
+    # needs: [test-linux, test-windows, test-macos]  # Uncomment when macOS is enabled
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - name: Build wheel and sdist
+        run: uv build
+      - name: Verify wheel installs cleanly
+        run: |
+          uv venv --clear /tmp/lef-release-test
+          uv pip install --python /tmp/lef-release-test/bin/python dist/*.whl
+          /tmp/lef-release-test/bin/python -c "import lef; print(f'LEF {lef.__version__}')"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+          retention-days: 90
+
+  publish-pypi:
+    name: Publish to PyPI
+    runs-on: [self-hosted, Linux]
+    needs: [build]
+    permissions:
+      contents: read
+      id-token: write  # Required for trusted publishing
+    environment:
+      name: pypi
+      url: https://pypi.org/project/lefx/
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # Uses trusted publishing (OIDC) — no API token needed
+        # Requires PyPI project to be configured for GitHub OIDC
+        # See: https://docs.pypi.org/trusted-publishers/
+
+  github-release:
+    name: GitHub Release
+    runs-on: [self-hosted, Linux]
+    needs: [build]
+    permissions:
+      contents: write  # Required to create releases
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true
+          name: "LEF ${{ github.ref_name }}"
+
+  # publish-macos:
+  #   # If you ever need macOS-specific wheels (e.g. for native extensions)
+  #   name: Build macOS Wheel
+  #   runs-on: [self-hosted, macOS]
+  #   needs: [test-macos]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: astral-sh/setup-uv@v4
+  #       with:
+  #         version: "latest"
+  #     - name: Build
+  #       run: uv build
+  #     - uses: actions/upload-artifact@v4
+  #       with:
+  #         name: dist-macos
+  #         path: dist/

lefx-0.3.0/.gitignore

@@ -0,0 +1,31 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.lef/
+dist/
+build/
+*.egg-info/
+*.egg
+.eggs/
+*.whl
+.venv/
+venv/
+env/
+.env
+.env.*
+!.env.example
+*.log
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+htmlcov/
+.coverage
+.coverage.*
+coverage.xml
+*.cover
+.hypothesis/
+.tox/
+.nox/
+.uv/
+uv.lock

lefx-0.3.0/CHANGELOG.md

@@ -0,0 +1,75 @@
+# Changelog
+
+All notable changes to LEF will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.3.0] - 2026-03-30
+
+### Changed
+- Renamed PyPI package from `lef` to `lefx` (import remains `import lef`)
+
+## [0.2.0] - 2026-03-28
+
+### Added
+- **Result export**: JSON, CSV, and JUnit XML export via `--output` flag and `export_results()` API
+- **Rich terminal output**: `format_results_table()` with per-metric summary, min/max/avg, and threshold pass/fail
+- **Git-aware metadata**: Auto-injects branch, commit SHA, author into experiment metadata; detects CI environments (GitHub Actions, Azure DevOps, GitLab CI, Jenkins)
+- **Baseline management**: `save_baseline()`, `load_baseline()`, `compare_results()` for regression detection; `lef baseline list/delete` CLI
+- **Experiment comparison**: `ComparisonReport` with table/markdown output and `compare_experiments()` for LangSmith experiments
+- **Result caching**: `ResultCache` with content-addressable hashing and TTL for avoiding re-invocation of expensive targets
+- **Watch mode**: `--watch` flag for auto-rerun on file changes during iterative development
+- **CI/CD PR comments**: `post_github_comment()` and `post_azdo_comment()` for posting results to GitHub and Azure DevOps PRs
+- **QA endpoint testing**: `lef qa` subcommand for testing deployed HTTP endpoints against datasets
+- **pytest plugin**: `lef_eval` fixture and config-file collection via `pytest11` entry point
+- **Synthetic data generation**: `generate_from_docs()`, `generate_from_traces()`, `generate_adversarial()`, `diversify_dataset()`
+- **Production monitoring**: `MonitorDaemon` for continuous evaluation of LangSmith runs with threshold alerting
+- **Red-team testing**: `run_redteam()` with 6 attack categories (prompt injection, jailbreak, PII extraction, hallucination inducement, toxicity, bias) and 3 built-in safety scorers
+- **Remote targets**: HTTP URLs supported as targets in config files via `create_remote_target()`
+- **File path targets**: Target functions can be referenced by file path (`/path/to/file.py:function`)
+- **Multi-config composition**: Pass multiple YAML configs to merge evaluators, thresholds, and metadata
+- **CLI subcommands**: `compare`, `baseline`, `qa`, `monitor`, `redteam`, `dataset` (pull/push/diff/generate)
+- **`--version` flag**: `lef -v` / `lef --version` shows version
+- `py.typed` marker for PEP 561 type information
+- 84 public API exports (up from 52)
+
+### Fixed
+- Outputs kwarg override bug in dual-signature resolution across decorators and base classes
+- `pass_rate` treating missing feedback as passing (now correctly treats as failing)
+- Client reuse in `online/tracing.py` (singleton default client avoids per-call overhead)
+- GitHub API URL for comment updates (was missing `/issues/` path segment)
+- Monitor `_seen_run_ids` used unordered set for eviction (now uses OrderedDict)
+- `compare_results()` now handles baseline dict format for both parameters
+- `_load_config()` rejects non-dict YAML (empty files, arrays) with clear error
+- Malformed config files produce clean error messages instead of raw tracebacks
+- Duplicate `-v`/`--verbose` flag between parent and run subparser
+- `lef` with no arguments now exits with code 2 (was incorrectly 0)
+
+### Changed
+- Version bumped to 0.2.0
+- Development Status classifier changed from Alpha to Beta
+- `-v` flag changed from `--verbose` to `--version` (use `--verbose` for debug logging)
+- `pyproject.toml` extras now use self-references to avoid duplicate version pins
+
+## [0.1.0] - 2024-12-01
+
+### Added
+- Core evaluation framework with `EvalResult`, `EvalResultBatch`, and `JudgeModel` types
+- `@scorer` and `@evaluator` decorators for creating LangSmith-compatible evaluators
+- Built-in scorers: `exact_match`, `contains`, `regex_match`, `json_match`
+- Custom scorer factories: `create_scorer`, `create_composite_scorer`
+- 14 pre-built LLM-as-Judge evaluators (correctness, safety, hallucination, RAG, etc.)
+- Custom judge creation via `create_judge()` with openevals integration
+- Agent trajectory evaluation via `create_trajectory_evaluator` and `create_trajectory_judge`
+- Dataset runner: `run_eval`, `arun_eval`, `EvalRunner`, `create_dataset`
+- Local dataset loading from YAML, JSON, and CSV files
+- Online evaluation: `evaluate_run`, `OnlineEvaluator`, `create_rule`
+- LangChain integration: `evaluate_chain`, `create_chain_target` (sync + async)
+- LangGraph integration: `evaluate_graph`, `create_graph_target` (sync + async)
+- CI/CD gating with `assert_scores` and `check_scores`
+- CLI tool (`lef run`) for running evaluation suites from config files
+- `LefConfig` for environment-based configuration
+- Dual-signature compatibility (legacy `run, example` and new-style kwargs)
+- 222 tests with full mocking (no API calls required)
+- Comprehensive README with quick-start examples and API reference

lefx-0.3.0/CLAUDE.md

@@ -0,0 +1,101 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What is this project?
+
+LEF (LangSmith Evaluation Framework) is a plug-and-play evaluation system for LangChain, LangGraph, and LangSmith projects. It wraps `langsmith`, `openevals`, and `agentevals` into a unified framework with built-in QA/CI support.
+
+## Common Commands
+
+```bash
+# Install dependencies
+uv sync --extra dev --extra all
+
+# Run all tests
+uv run pytest
+
+# Run a single test file
+uv run pytest tests/test_scorers.py
+
+# Run a single test by name
+uv run pytest tests/test_scorers.py -k "test_exact_match"
+
+# Lint
+uv run ruff check src/ tests/
+
+# Lint with auto-fix
+uv run ruff check --fix src/ tests/
+
+# Type check
+uv run mypy src/
+
+# Run the CLI
+uv run lef run <config.yaml>
+```
+
+## Architecture
+
+Source lives in `src/lef/`. All public API is exported from `src/lef/__init__.py` via `__all__`.
+
+### Key design decisions
+
+- **EvalResult is a `dict` subclass** (not a Pydantic model) — this is required because LangSmith's `evaluate()` expects dict-like results. It also provides property access (`.key`, `.score`, `.comment`, `.metadata`).
+- **Dual-signature compatibility** — all evaluators accept both LangSmith's legacy `(run, example)` and new-style `(inputs=, outputs=, reference_outputs=)` keyword signatures. The `@scorer` decorator in `core/decorators.py` handles this normalization.
+- **Decorators normalize return types** — `@scorer` accepts `bool`, `float`, `int`, `dict`, or `EvalResult` returns and always produces a LangSmith-compatible dict.
+- **openevals judges are thin wrappers** — `judges/llm.py` factory functions (e.g., `correctness_judge()`) wrap `openevals.create_llm_as_judge()` with LEF defaults and return callables.
+- **Local datasets don't need LangSmith** — `load_examples()` from `datasets/loader.py` supports YAML/JSON/CSV. Combined with `upload_results=False`, evaluation runs fully offline.
+
+### Module layout
+
+**Core:**
+- `core/types.py` — `EvalResult`, `EvalResultBatch`, `JudgeModel` enum, evaluator protocols
+- `core/base.py` — `BaseEvaluator`, `AsyncBaseEvaluator` abstract classes
+- `core/decorators.py` — `@scorer` and `@evaluator` decorators
+
+**Scorers & Judges:**
+- `scorers/builtin.py` — Rule-based scorers: `exact_match`, `contains`, `regex_match`, `json_match`, `mean_score`, `pass_rate`
+- `scorers/custom.py` — `create_scorer`, `create_composite_scorer` factories
+- `judges/llm.py` — 20+ LLM-as-judge factories (correctness, safety, hallucination, RAG, etc.)
+- `judges/trajectory.py` — Agent trajectory evaluators wrapping `agentevals`
+
+**Data & Runner:**
+- `datasets/runner.py` — `run_eval`, `arun_eval`, `EvalRunner`, `create_dataset`
+- `datasets/loader.py` — `load_examples` for local YAML/JSON/CSV files
+
+**CLI:**
+- `cli.py` — CLI entry point with 7 subcommands: `run`, `compare`, `baseline`, `qa`, `monitor`, `redteam`, `dataset`
+
+**Integrations:**
+- `integrations/langchain.py` — LangChain chain evaluation
+- `integrations/langgraph.py` — LangGraph graph evaluation
+- `integrations/remote.py` — HTTP endpoint targets (`create_remote_target`)
+
+**v0.2.0 Modules:**
+- `output.py` — Rich terminal tables + JSON/CSV/JUnit XML export
+- `git_context.py` — Git-aware experiment metadata (branch, commit, CI detection)
+- `compare.py` — Baseline management and experiment comparison
+- `cache.py` — Content-addressable result caching with TTL
+- `watch.py` — File-watching mode for iterative development
+- `ci/github.py` — GitHub PR comment integration
+- `ci/azuredevops.py` — Azure DevOps PR comment integration
+- `pytest_plugin.py` — pytest plugin with `lef_eval` fixture (entry point: `pytest11`)
+- `synthetic.py` — Synthetic dataset generation from documents/traces
+- `monitor.py` — Production monitoring daemon with threshold alerting
+- `redteam.py` — Adversarial red-team testing with 6 attack categories
+- `online/tracing.py` — Online evaluation of production LangSmith runs
+
+## Testing
+
+- All tests in `tests/`, using pytest with `asyncio_mode = "auto"` (no need for `@pytest.mark.asyncio`)
+- Tests mock LLM APIs via `unittest.mock.patch` — no real API calls needed
+- Shared fixtures in `tests/conftest.py`: `sample_inputs`, `sample_outputs`, `sample_reference_outputs`, `wrong_outputs`, `empty_outputs`
+
+## When Adding New Evaluators
+
+1. Add the factory function to `src/lef/judges/llm.py` (for LLM judges) or `src/lef/scorers/builtin.py` (for rule-based)
+2. Export from the sub-package `__init__.py`
+3. Export from the top-level `src/lef/__init__.py` and add to `__all__`
+4. Add to `_BUILTIN_EVALUATORS` in `src/lef/cli.py` for CLI support
+5. Add tests in `tests/`
+6. Update the README table

lefx-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 bogware
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lefx-0.3.0/MANIFEST.in

@@ -0,0 +1,5 @@
+include LICENSE
+include README.md
+include CHANGELOG.md
+include pyproject.toml
+recursive-include src/lef *.py