python-flexeval 0.1.5__tar.gz

python_flexeval-0.1.5/.env-example

@@ -0,0 +1,5 @@
+
+OPENAI_API_KEY=sk-***
+EVALS_THREADS=5
+EVALS_SEQUENTIAL=0
+EVALS_THREAD_TIMEOUT=600

python_flexeval-0.1.5/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+
+- package-ecosystem: "uv"
+  directory: "/"
+  schedule:
+    interval: "weekly"
+  target-branch: "dev"

python_flexeval-0.1.5/.github/workflows/deploy-to-pypi.yml

@@ -0,0 +1,80 @@
+name: Publish package to PyPI and TestPyPI
+
+on: 
+  push:
+    branches: ["main"]
+    tags: ["v*"]
+  workflow_dispatch:
+
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        version: "latest"
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+    - name: Build a binary wheel and a source tarball
+      run: >-
+        uv build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python distribution to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/python-flexeval
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  publish-to-testpypi:
+    name: Publish Python distribution to TestPyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/python-flexeval
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/

python_flexeval-0.1.5/.github/workflows/github-pages.yml

@@ -0,0 +1,68 @@
+# Workflow for building and deploying a Sphinx site to GitHub Pages
+name: Deploy Sphinx site with GitHub Pages dependencies preinstalled
+
+on:
+  # Runs on pushes targeting following list of branches
+  push:
+    branches: ["main"]
+
+  # Also allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install docs dependencies
+        run: |
+          uv sync --group docs
+
+      - name: Build with Sphinx
+        run: |
+          make html BUILDDIR=_site
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: _site/html/
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

python_flexeval-0.1.5/.github/workflows/validate.yaml

@@ -0,0 +1,32 @@
+name: Run validation
+
+on: [push]
+
+jobs:
+  build:
+    name: Run validation
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install Python dependencies
+        run: |
+          uv sync
+
+      - name: Run automated validation checks
+        run: |
+          uv run python -m unittest discover -s tests.unit
+        env:
+          CURRENT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}

python_flexeval-0.1.5/.gitignore

@@ -0,0 +1,187 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+#Evals-specific 
+logs/*
+!logs/.gitkeep
+
+data/results/evals-outputs/*
+!data/results/evals-outputs/.gitkeep
+
+.DS_Store
+temp/
+test-cases/
+src/llm-evals/evals_sync/
+*.db
+*.db-*
+
+# notebooks
+*.ipynb
+
+# sqlite data
+*.sqlite
+
+# Claude Code
+.claude/
+
+# Docs
+docs/generated
+

python_flexeval-0.1.5/.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.12.3  # Ruff version
+    hooks:
+      - id: ruff-check # Run the linter
+        args: [ --fix ]
+      - id: ruff-format  # Run the formatter

python_flexeval-0.1.5/.python-version

@@ -0,0 +1 @@
+3.10

python_flexeval-0.1.5/.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./tests",
+        "-p",
+        "test_*.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true
+}

python_flexeval-0.1.5/CITATION.bib

@@ -0,0 +1,19 @@
+@inproceedings{christie_flexeval_2024,
+  author       = {S. Thomas Christie and
+                  Baptiste Moreau-Pernet and
+                  Yu Tian and
+                  John Whitmer},
+  title        = {FlexEval: a customizable tool for chatbot
+                   performance evaluation and dialogue analysis
+                  },
+  booktitle    = {Proceedings of the 17th International Conference
+                   on Educational Data Mining
+                  },
+  year         = 2024,
+  pages        = {903--908},
+  publisher    = {International Educational Data Mining Society},
+  month        = jul,
+  venue        = {Atlanta, Georgia, USA},
+  doi          = {10.5281/zenodo.12729993},
+  url          = {https://doi.org/10.5281/zenodo.12729993},
+}

python_flexeval-0.1.5/CITATION.cff

@@ -0,0 +1,37 @@
+cff-version: 1.2.0
+message: 'If FlexEval contributes to a project that leads to a scientific publication, please acknowledge this fact by citing: S. Thomas Christie, Baptiste Moreau-Pernet, Yu Tian, and John Whitmer, "FlexEval: a customizable tool for chatbot performance evaluation and dialogue analysis", Proceedings of the 17th International Conference on Educational Data Mining, pp. 903-908, 2024.'
+title: 'FlexEval: a customizable tool for chatbot performance evaluation and dialogue analysis'
+authors:
+  - family-names: Christie
+    given-names: S. Thomas
+  - family-names: Moreau-Pernet
+    given-names: Baptiste
+  - family-names: Tian
+    given-names: Yu
+  - family-names: Whitmer
+    given-names: John
+type: software
+url: 'https://github.com/DigitalHarborFoundation/FlexEval'
+repository-code: 'https://github.com/DigitalHarborFoundation/FlexEval'
+preferred-citation:
+  type: conference-paper
+  authors:
+    - family-names: Christie
+      given-names: S. Thomas
+    - family-names: Moreau-Pernet
+      given-names: Baptiste
+    - family-names: Tian
+      given-names: Yu
+    - family-names: Whitmer
+      given-names: John
+  title: "FlexEval: a customizable tool for chatbot performance evaluation and dialogue analysis"
+  year: 2024
+  date-published: 2024-07
+  conference: 17th International Conference on Educational Data Mining
+  venue: Atlanta, Georgia, USA
+  start: 903
+  end: 908
+  doi: 10.5281/zenodo.12729993
+  publisher:
+    name: International Educational Data Mining Society
+    website: 'https://educationaldatamining.org/'

python_flexeval-0.1.5/CLAUDE.md

@@ -0,0 +1,114 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Development Commands
+
+### Environment Setup
+```bash
+# Install uv package manager first: https://docs.astral.sh/uv/getting-started/installation/
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Install dependencies
+uv sync
+```
+
+### Running FlexEval
+
+Not quite ready; our refactor is close to being done.
+
+```bash
+# CLI help
+uv run python -m flexeval --help
+```
+
+### Testing
+```bash
+# Run unit tests
+uv run python -m unittest discover -s tests.unit
+
+# Run specific test file
+uv run python -m unittest tests.unit.{module_name}
+
+# Integration tests are in tests/integration/, but aren't ready yet
+```
+
+### Build and Dependencies
+```bash
+# Build the package
+uv build
+
+# Add dependency
+uv add {package_name}
+
+# Update dependencies
+uv lock --upgrade
+```
+
+### Linting
+```bash
+# Run ruff linter (configured in ruff.toml)
+ruff check src/
+ruff format src/
+```
+
+## Architecture Overview
+
+FlexEval is a tool for evaluating LLM-powered systems using custom metrics, completion functions, and LLM-graded rubrics. The system operates on conversational data at multiple granularities.
+
+### Core Abstractions
+
+**EvalRun** (`src/flexeval/schema/evalrun_schema.py`): The top-level execution unit that combines:
+- Data sources (conversations in JSONL format as inputs, an SQLite filepath as output)
+- An Eval specification (metrics to compute)
+- Configuration (workers, database path, etc.)
+- Rubric and function sources
+
+**Eval** (`src/flexeval/schema/eval_schema.py`): Defines what to evaluate:
+- Function metrics (Python functions that compute numeric values)
+- Rubric metrics (LLM-graded evaluations using chain-of-thought)
+- Completion LLM (for generating new responses)
+- Grader LLM (for rubric evaluation)
+- Dependencies between metrics
+
+**Config** (`src/flexeval/schema/config_schema.py`): Defines how to evaluate (e.g. single- vs multi-process, etc.)
+
+### Data Hierarchy
+The evaluation operates at multiple levels of granularity:
+- **Thread**: Full conversation
+- **Turn**: User-assistant exchange pair  
+- **Message**: Individual message from user or assistant
+- **ToolCall**: Function/tool invocation within a message
+
+### Key Components
+
+**Configuration System**:
+- `src/flexeval/configuration/rubric_metrics.yaml`: Default set of rubrics
+- `src/flexeval/configuration/function_metrics.py`: Default set of Python metric functions
+- `src/flexeval/configuration/completion_functions.py`: Set of functions for producing LLM completions (usually via API call)
+
+**Execution Pipeline** (`src/flexeval/runner.py`):
+1. Load configuration and eval specification
+2. Create Dataset from data sources
+3. Run EvalRunner to compute metrics
+4. Store results in SQLite database
+
+**Metric System**:
+- Function metrics: Python functions that analyze conversations/turns/messages
+- Rubric metrics: LLM-based evaluations using structured prompts
+- Dependencies: Metrics can depend on other metrics meeting certain criteria
+- Aggregation: Results aggregated by role, turn, etc.
+
+### Data Format
+
+Input data is in JSONL format with conversations as:
+```json
+{"input": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]}
+```
+
+Results are stored in an SQLite database (defaulting to `data/results/results.db`) for querying and analysis.
+
+### Schema System
+
+The project uses Pydantic models for validation:
+- `src/flexeval/schema/`: Contains all schema definitions