python-flexeval 0.1.5__tar.gz → 0.2.0__tar.gz

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/.gitignore

@@ -184,4 +184,5 @@ src/llm-evals/evals_sync/
 
 # Docs
 docs/generated
+jupyter_execute/
 

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/Makefile

@@ -6,7 +6,7 @@ SPHINXBUILD   ?= uv run sphinx-build
 SOURCEDIR     = docs
 BUILDDIR      = build
 
-.PHONY: dochelp docautobuild docclean unittest Makefile
+.PHONY: dochelp docautobuild docclean unittest get-version set-version Makefile
 
 # Put it first so that "make" without argument is like "make dochelp".
 dochelp:
@@ -23,6 +23,16 @@ docclean:
 unittest:
 	@uv run python -m unittest discover -s tests.unit
 
+# see: https://hatch.pypa.io/1.13/version/
+get-version:
+	@uv run hatch version
+
+set-version:
+	@if [ -z "$(VERSION)" ]; then \
+		echo "VERSION is not set. Usage: make set-version VERSION=x.y.z"; \
+		exit 1; \
+	fi
+	@uv run hatch version "$(VERSION)"
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-flexeval
-Version: 0.1.5
+Version: 0.2.0
 Summary: FlexEval is a tool for designing custom metrics, completion functions, and LLM-graded rubrics for evaluating the behavior of LLM-powered systems.
 Project-URL: Homepage, https://digitalharborfoundation.github.io/FlexEval/
 Project-URL: GitHub, https://github.com/DigitalHarborFoundation/FlexEval
@@ -40,10 +40,12 @@ Description-Content-Type: text/markdown
 
 # FlexEval LLM Evals
 
+[![PyPi](https://img.shields.io/pypi/v/python-flexeval)](https://pypi.org/project/python-flexeval/)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.12729993.svg)](https://doi.org/10.5281/zenodo.12729993)
 [![License](https://img.shields.io/github/license/DigitalHarborFoundation/FlexEval)](https://github.com/DigitalHarborFoundation/FlexEval/blob/main/LICENSE)
+[![GitHub issues](https://img.shields.io/badge/issue_tracking-github-blue.svg)](https://github.com/DigitalHarborFoundation/FlexEval/issues)
 
-![FlexEval banner](/docs/_static/flexeval_banner.svg)
+![FlexEval banner](https://raw.githubusercontent.com/DigitalHarborFoundation/FlexEval/refs/heads/main/docs/_static/flexeval_banner.svg)
 
 FlexEval is a tool for designing custom metrics, completion functions, and LLM-graded rubrics for evaluating the behavior of LLM-powered systems.
 
@@ -73,7 +75,7 @@ flexeval.run(eval_run)
 
 This example computes [Flesch reading ease](https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch_reading_ease) for every turn in a list of conversations provided in JSONL format. The metric values are stored in an SQLite database called `eval_results.db`.
 
-See additional usage examples in the [vignettes](/vignettes).
+See additional usage examples in the [vignettes](https://github.com/DigitalHarborFoundation/FlexEval/tree/main/vignettes).
 
 ## Installation
 
@@ -97,7 +99,7 @@ FlexEval is designed to be "batteries included" for many basic use cases. It sup
 - a set of useful rubrics
 - a set of useful Python functions
 
-Evaluation results are saved in an SQLite database. See the [Metric Analysis](/vignettes/metric_analysis.ipynb) vignette for a sample analysis demonstrating the structure and utility of the data saved by FlexEval.
+Evaluation results are saved in an SQLite database. See the [Metric Analysis](https://digitalharborfoundation.github.io/FlexEval/generated/vignettes/metric_analysis.html) vignette for a sample analysis demonstrating the structure and utility of the data saved by FlexEval.
 
 
 Read more in the [Getting Started](https://digitalharborfoundation.github.io/FlexEval/getting_started.html) guide.
@@ -115,4 +117,4 @@ Pull requests are welcome. Feel free to contribute:
 - Bug fixes
 - New features
 
-See [DEVELOPMENT.md](DEVELOPMENT.md).
+See [DEVELOPMENT.md](https://github.com/DigitalHarborFoundation/FlexEval/tree/main/DEVELOPMENT.md).

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/README.md

@@ -1,9 +1,11 @@
 # FlexEval LLM Evals
 
+[![PyPi](https://img.shields.io/pypi/v/python-flexeval)](https://pypi.org/project/python-flexeval/)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.12729993.svg)](https://doi.org/10.5281/zenodo.12729993)
 [![License](https://img.shields.io/github/license/DigitalHarborFoundation/FlexEval)](https://github.com/DigitalHarborFoundation/FlexEval/blob/main/LICENSE)
+[![GitHub issues](https://img.shields.io/badge/issue_tracking-github-blue.svg)](https://github.com/DigitalHarborFoundation/FlexEval/issues)
 
-![FlexEval banner](/docs/_static/flexeval_banner.svg)
+![FlexEval banner](https://raw.githubusercontent.com/DigitalHarborFoundation/FlexEval/refs/heads/main/docs/_static/flexeval_banner.svg)
 
 FlexEval is a tool for designing custom metrics, completion functions, and LLM-graded rubrics for evaluating the behavior of LLM-powered systems.
 
@@ -33,7 +35,7 @@ flexeval.run(eval_run)
 
 This example computes [Flesch reading ease](https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch_reading_ease) for every turn in a list of conversations provided in JSONL format. The metric values are stored in an SQLite database called `eval_results.db`.
 
-See additional usage examples in the [vignettes](/vignettes).
+See additional usage examples in the [vignettes](https://github.com/DigitalHarborFoundation/FlexEval/tree/main/vignettes).
 
 ## Installation
 
@@ -57,7 +59,7 @@ FlexEval is designed to be "batteries included" for many basic use cases. It sup
 - a set of useful rubrics
 - a set of useful Python functions
 
-Evaluation results are saved in an SQLite database. See the [Metric Analysis](/vignettes/metric_analysis.ipynb) vignette for a sample analysis demonstrating the structure and utility of the data saved by FlexEval.
+Evaluation results are saved in an SQLite database. See the [Metric Analysis](https://digitalharborfoundation.github.io/FlexEval/generated/vignettes/metric_analysis.html) vignette for a sample analysis demonstrating the structure and utility of the data saved by FlexEval.
 
 
 Read more in the [Getting Started](https://digitalharborfoundation.github.io/FlexEval/getting_started.html) guide.
@@ -75,4 +77,4 @@ Pull requests are welcome. Feel free to contribute:
 - Bug fixes
 - New features
 
-See [DEVELOPMENT.md](DEVELOPMENT.md).
+See [DEVELOPMENT.md](https://github.com/DigitalHarborFoundation/FlexEval/tree/main/DEVELOPMENT.md).

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/docs/conf.py

@@ -89,6 +89,10 @@ intersphinx_mapping = {
     "pydantic": ("https://docs.pydantic.dev/latest", None),
 }
 
+docutils_conf = {
+    "line-length-limit": None,  # Disable docutils line-length limit
+}
+
 
 def linkcode_resolve(domain, info):
     """
@@ -159,6 +163,10 @@ myst_enable_extensions = [
 ]
 myst_url_schemes = ("http", "https", "mailto")
 
+# myst-nb configuration
+nb_execution_mode = "off"  # Don't re-execute, use existing outputs
+nb_merge_streams = True
+
 autosummary_generate = True
 autodoc_typehints = "signature"
 autodoc_default_options = {

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/docs/index.rst

@@ -1,11 +1,12 @@
-.. FlexEval documentation master file, created by
-   sphinx-quickstart on Thu Jul  3 12:21:33 2025.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
+.. FlexEval documentation master file, originally created by sphinx-quickstart on 2025 July 3 12:21:33.
 
 FlexEval documentation
 ======================
 
+.. image:: https://img.shields.io/pypi/v/python-flexeval
+   :target: https://pypi.org/project/python-flexeval/
+   :alt: PyPI
+
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.12729993.svg
    :target: https://doi.org/10.5281/zenodo.12729993
    :alt: Zenodo DOI
@@ -14,6 +15,10 @@ FlexEval documentation
    :target: https://github.com/DigitalHarborFoundation/FlexEval/blob/main/LICENSE
    :alt: FlexEval license
 
+.. image:: https://img.shields.io/badge/issue_tracking-github-blue.svg
+   :target: https://github.com/DigitalHarborFoundation/FlexEval/issues
+   :alt: Issue tracking on GitHub
+
 .. raw:: html
 
    <br>

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/docs/vignettes.py

@@ -44,7 +44,7 @@ def extract_vignette_path_strings(file_contents: str) -> list[str]:
 
 def write_if_changed(path: Path, old_content: str, new_content: str):
     if path.exists():
-        old_content = path.read_text()
+        # check if the file contents are different
         if old_content == new_content:
             return  # No change — don't touch file
     path.write_text(new_content)
@@ -67,33 +67,30 @@ def generate_custom_stubs(app):
 
     for vignette_file in vignettes_dir.glob("*"):
         stem = vignette_file.stem
-        rst_file = output_dir / f"{stem}.rst"
+        target_file = output_dir / f"{stem}.rst"
+        if vignette_file.suffix == ".ipynb":
+            target_file = output_dir / f"{stem}.ipynb"
 
         current_contents = ""
-        if rst_file.exists():
-            current_contents = rst_file.read_text()
+        if target_file.exists():
+            current_contents = target_file.read_text()
 
         if vignette_file.suffix == ".py":
-            generate_python_stub(src_dir, vignette_file, rst_file, current_contents)
+            generate_python_stub(src_dir, vignette_file, target_file, current_contents)
         elif vignette_file.suffix == ".ipynb":
-            generate_ipynb_stub(src_dir, vignette_file, rst_file, current_contents)
+            generate_ipynb_stub(vignette_file, target_file, current_contents)
         elif vignette_file.suffix == ".md":
-            generate_md_stub(src_dir, vignette_file, rst_file, current_contents)
+            generate_md_stub(src_dir, vignette_file, target_file, current_contents)
         else:
             logger.info(
                 f"Unsupported file type {vignette_file.suffix}; skipping while creating vignette stubs."
             )
 
 
-def generate_ipynb_stub(
-    src_dir: Path, ipynb_file: Path, rst_file: Path, current_contents: str
-):
-    new_contents = f""".. _{ipynb_file.stem}:
-
-.. include:: ../../../{ipynb_file.relative_to(src_dir.parent)}
-   :parser: myst_nb.docutils_
-"""
-    write_if_changed(rst_file, current_contents, new_contents)
+def generate_ipynb_stub(ipynb_file: Path, target_file: Path, current_contents: str):
+    # unlike the other file types, we just copy ipynb files
+    new_contents = ipynb_file.read_text()
+    write_if_changed(target_file, current_contents, new_contents)
 
 
 def generate_md_stub(

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "python-flexeval"
-version = "0.1.5"
+dynamic = ["version"]
 description = "FlexEval is a tool for designing custom metrics, completion functions, and LLM-graded rubrics for evaluating the behavior of LLM-powered systems."
 authors = [
   { name="S. Thomas Christie" },
@@ -52,6 +52,7 @@ Issues = "https://github.com/DigitalHarborFoundation/FlexEval/issues"
 
 [dependency-groups]
 dev = [
+    "hatch>=1.14.1",
     "jupyter>=1.1.1",
     "matplotlib>=3.10.3",
     "pre-commit>=4.2.0",
@@ -81,3 +82,6 @@ build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/flexeval"]
+
+[tool.hatch.version]
+path = "src/flexeval/__about__.py"

python_flexeval-0.2.0/src/flexeval/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/__init__.py

@@ -4,8 +4,10 @@ This top-level import exposes the :func:`~flexeval.runner.run` method."""
 
 from flexeval import metrics
 from flexeval.runner import run
+from .__about__ import __version__
 
 __all__ = [
     "metrics",
     "run",
+    "__version__",
 ]

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/classes/eval_runner.py

@@ -111,7 +111,7 @@ class EvalRunner:
 
     def load_evaluation_settings(self):
         """This function parses our eval suite and puts it in the data structure we'll need
-        for easy use at run-time
+        for easy use at run-time.
         """
         # if the current eval has a 'config' entry, overwrite configuration options with its entries
         if (

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/compute_metrics.py

@@ -42,7 +42,7 @@ class ObjectMetric:
 
 
 class MetricGraphBuilder:
-    """Builds :class:`networkx.DiGraph`\s of :class:`~flexeval.compute_metrics.ObjectMetric` instances that reflect any computational dependencies between them."""
+    """Builds :class:`networkx.DiGraph` s of :class:`~flexeval.compute_metrics.ObjectMetric` instances that reflect any computational dependencies between them."""
 
     def __init__(self):
         # key: tuple(metric_level, metric_id, object_id)

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/configuration/function_metrics.py

@@ -122,8 +122,8 @@ def is_role(object: Union[Turn, Message], role: str) -> dict:
     and 0 otherwise.
 
     Args:
-    object: the Turn or Message
-    role: a string with the desired role to check against
+        object: the Turn or Message
+        role: a string with the desired role to check against
     """
     return {role: int(object.role == role)}
 

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/db_utils.py

@@ -14,6 +14,11 @@ from flexeval.classes.turn import Turn
 DATABASE_TABLES = [EvalSetRun, Dataset, Thread, Turn, Message, ToolCall, Metric]
 
 
+def ensure_database(database_path: str):
+    if not classes_base.database.is_connection_usable():
+        initialize_database(database_path)
+
+
 def initialize_database(database_path: str, clear_tables: bool = False):
     classes_base.database.init(database_path)
     # classes_base.database.start()
@@ -34,5 +39,7 @@ def bind_to_database(database_path: str) -> pw.Database:
     new_database = classes_base.create_sqlite_database(database_path)
     new_database.bind(DATABASE_TABLES)
     # Verify the binding worked by checking one of the models
-    assert classes_base.BaseModel._meta.database == new_database
+    assert classes_base.BaseModel._meta.database == new_database, (
+        f"Binding to '{database_path}' failed."
+    )
     return new_database

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/metrics/__init__.py

@@ -1,4 +1,4 @@
-"""Utility functions for accessing metrics."""
+"""Utility functions for working with metrics."""
 
 from flexeval.metrics import access, save
 

python_flexeval-0.2.0/src/flexeval/metrics/access.py

@@ -0,0 +1,50 @@
+"""Utility functions for accessing metrics."""
+
+from collections import Counter
+
+from flexeval.classes import metric, message, turn, thread
+
+
+def count_dict_values(lst: list[dict]) -> dict[str, Counter]:
+    """Convenience function for counting key values.
+
+    Args:
+        lst (list[dict]): List of dictionaries.
+
+    Returns:
+        dict[str, Counter]: counter for each key that appears in the dicts in lst.
+    """
+    counts = {}
+    for d in lst:
+        for k, v in d.items():
+            if k not in counts:
+                counts[k] = Counter()
+            counts[k][v] += 1
+    return counts
+
+
+def get_all_metrics() -> list[dict]:
+    results = []
+    for m in metric.Metric.select():
+        results.append(m.__data__.copy())
+    return results
+
+
+def get_first_user_message_for_threads(thread_ids: set) -> list[dict]:
+    """Get the first user message in each thread.
+
+    Args:
+        thread_ids (set): The set of thread IDs to retrieve messages for.
+
+    Returns:
+        list[dict]: An iterable of messages.
+    """
+    return (
+        message.Message.select()
+        .join(thread.Thread)
+        .where(thread.Thread.id.in_(thread_ids))
+        .where(message.Message.role == "user")
+        .join(turn.Turn)
+        .where(turn.Turn.index_in_thread == 0)
+        .dicts()
+    )

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/schema/eval_schema.py

@@ -16,6 +16,8 @@ MetricLevel = Literal["Message", "Turn", "Thread", "ToolCall"]
 
 
 class DependsOnItem(BaseModel):
+    """Defines a metric dependency."""
+
     class Config:
         extra = "forbid"
 
@@ -56,6 +58,8 @@ class DependsOnItem(BaseModel):
 
 
 class MetricItem(BaseModel):
+    "Defines a metric."
+
     name: str = Field(
         ...,
         description="The function to call or name of rubric to use to compute this metric.",
@@ -72,6 +76,8 @@ class MetricItem(BaseModel):
 
 
 class FunctionItem(MetricItem):
+    """Defines a metric computed from a Python function."""
+
     kwargs: schema_utils.OptionalDict = Field(
         default_factory=dict,
         description="Keyword arguments for the function. Each key must correspond to an argument in the function. Extra keys will cause an error.",
@@ -80,6 +86,8 @@ class FunctionItem(MetricItem):
 
 
 class RubricItem(MetricItem):
+    """Defines a metric computed from a rubric."""
+
     # TODO is RubricItem.kwargs actually used?
     kwargs: Optional[Dict[str, Any]] = Field(
         default_factory=dict,
@@ -115,6 +123,8 @@ class CompletionLlm(BaseModel):
 
 
 class GraderLlm(BaseModel):
+    """Defines the LLM used for evaluating rubrics."""
+
     class Config:
         extra = "forbid"
 

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/schema/evalrun_schema.py

@@ -37,7 +37,7 @@ class FileDataSource(DataSource):
 
 
 class FunctionsCollection(BaseModel):
-    """Collection of functions that can be used as :class:`~flexeval.schema.eval_schema.FunctionItem`\s."""
+    """Collection of functions that can be used as :class:`~flexeval.schema.eval_schema.FunctionItem` s."""
 
     functions: list[Callable] = Field(
         default_factory=list,

{python_flexeval-0.1.5 → python_flexeval-0.2.0}/src/flexeval/schema/rubric_schema.py

@@ -32,7 +32,7 @@ class Rubric(BaseModel):
 
 
 class RubricsCollection(BaseModel):
-    """Collection of rubrics that can be used as :class:`~flexeval.schema.eval_schema.RubricItem`\s."""
+    """Collection of rubrics that can be used as :class:`~flexeval.schema.eval_schema.RubricItem` s."""
 
     rubrics: dict[str, Rubric] = Field(
         default_factory=dict,