lmnr 0.4.10__tar.gz → 0.4.12__tar.gz

{lmnr-0.4.10 → lmnr-0.4.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.10
+Version: 0.4.12
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -11,9 +11,9 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: argparse (>=1.0,<2.0)
 Requires-Dist: asyncio (>=3.0,<4.0)
 Requires-Dist: backoff (>=2.0,<3.0)
-Requires-Dist: colorama (>=0.4,<0.5)
 Requires-Dist: deprecated (>=1.0,<2.0)
 Requires-Dist: jinja2 (>=3.0,<4.0)
 Requires-Dist: opentelemetry-api (>=1.27.0,<2.0.0)
@@ -54,111 +54,123 @@ Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: python-dotenv (>=1.0,<2.0)
 Requires-Dist: requests (>=2.0,<3.0)
 Requires-Dist: tenacity (>=8.0,<9.0)
+Requires-Dist: tqdm (>=4.0,<5.0)
 Description-Content-Type: text/markdown
 
 # Laminar Python
 
-OpenTelemetry log sender for [Laminar](https://github.com/lmnr-ai/lmnr) for Python code.
+Python SDK for [Laminar](https://www.lmnr.ai).
+
+[Laminar](https://www.lmnr.ai) is an open-source platform for engineering LLM products. Trace, evaluate, annotate, and analyze LLM data. Bring LLM applications to production with confidence.
+
+Check our [open-source repo](https://github.com/lmnr-ai/lmnr) and don't forget to star it ⭐
 
  <a href="https://pypi.org/project/lmnr/"> ![PyPI - Version](https://img.shields.io/pypi/v/lmnr?label=lmnr&logo=pypi&logoColor=3775A9) </a>
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/lmnr)
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/lmnr)
 
 
-
 ## Quickstart
-```sh
-python3 -m venv .myenv
-source .myenv/bin/activate  # or use your favorite env management tool
 
+First, install the package:
+
+```sh
 pip install lmnr
 ```
 
-And the in your main Python file
+And then in the code
 
 ```python
-from lmnr import Laminar as L, Instruments
+from lmnr import Laminar as L
 
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
+L.initialize(project_api_key="<PROJECT_API_KEY>")
 ```
 
-If you want to automatically instrument particular LLM, Vector DB, and related
-calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
-
-You can pass an empty set as `instruments=set()` to disable any kind of automatic instrumentation. 
-Also if you want to automatically instrument all supported libraries, then pass `instruments=None` or don't pass `instruments` at all.
+This will automatically instrument most of the LLM, Vector DB, and related
+calls with OpenTelemetry-compatible instrumentation.
 
-Our code is based on the [OpenLLMetry](https://github.com/traceloop/openllmetry), open-source package
-by TraceLoop. Also, we are grateful to Traceloop for implementing autoinstrumentations for many libraries.
-
-### Project API key
-
-Get the key from the settings page of your Laminar project ([Learn more](https://docs.lmnr.ai/api-reference/introduction#authentication)).
-You can either pass it to `.initialize()` or set it to `.env` at the root of your package with the key `LMNR_PROJECT_API_KEY`.
+Note that you need to only initialize Laminar once in your application.
 
 ## Instrumentation
 
-In addition to automatic instrumentation, we provide a simple `@observe()` decorator, if you want more fine-grained tracing
-or to trace other functions.
+### Manual instrumentation
 
-### Example
+To instrument any function in your code, we provide a simple `@observe()` decorator.
+This can be useful if you want to trace a request handler or a function which combines multiple LLM calls.
 
 ```python
 import os
 from openai import OpenAI
+from lmnr import Laminar as L, Instruments
 
-
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])
 
 client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
-@observe()  # annotate all functions you want to trace
-def poem_writer(topic="turbulence"):
+def poem_writer(topic: str):
     prompt = f"write a poem about {topic}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt},
+    ]
+
+    # OpenAI calls are still automatically instrumented
     response = client.chat.completions.create(
         model="gpt-4o",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": prompt},
-        ],
+        messages=messages,
     )
     poem = response.choices[0].message.content
+
     return poem
 
-print(poem_writer(topic="laminar flow"))
+@observe()
+def generate_poems():
+    poem1 = poem_writer(topic="laminar flow")
+    L.event("is_poem_generated", True)
+    poem2 = poem_writer(topic="turbulence")
+    L.event("is_poem_generated", True)
+    poems = f"{poem1}\n\n---\n\n{poem2}"
+    return poems
 ```
 
-### Manual instrumentation
-
-Also, you can `Laminar.start_as_current_span` if you want to record a chunk of your code.
+Also, you can use `Laminar.start_as_current_span` if you want to record a chunk of your code using `with` statement.
 
 ```python
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+def handle_user_request(topic: str):
+    with L.start_as_current_span(name="poem_writer", input=topic):
+        ...
+
+        poem = poem_writer(topic=topic)
+        
+        ...
+        
+        # while within the span, you can attach laminar events to it
+        L.event("is_poem_generated", True)
 
-def poem_writer(topic="turbulence"):
-    prompt = f"write a poem about {topic}"
-    messages = [
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": prompt},
-    ]
+        # Use set_span_output to record the output of the span
+        L.set_span_output(poem)
+```
 
-    with L.start_as_current_span(name="poem_writer", input=messages):
-        # OpenAI calls are still automatically instrumented with OpenLLMetry
-        response = client.chat.completions.create(
-            model="gpt-4o",
-            messages=messages,
-        )
-        poem = response.choices[0].message.content
-        # while within the span, you can attach laminar events to it
-        L.event("event_name", "event_value")
+### Automatic instrumentation
+
+Laminar allows you to automatically instrument majority of the most popular LLM, Vector DB, database, requests, and other libraries.
+
+If you want to automatically instrument a default set of libraries, then simply do NOT pass `instruments` argument to `.initialize()`.
+See the full list of available instrumentations in the [enum](/src/lmnr/traceloop_sdk/instruments.py).
 
-        L.set_span_output(poem) # set an output
+If you want to automatically instrument only specific LLM, Vector DB, or other
+calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+For example, if you want to only instrument OpenAI and Anthropic, then do the following:
+
+```python
+from lmnr import Laminar as L, Instruments
 
-        return poem
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"], instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
 ```
 
+If you want to fully disable any kind of autoinstrumentation, pass an empty set as `instruments=set()` to `.initialize()`. 
+
+Autoinstrumentations are provided by Traceloop's [OpenLLMetry](https://github.com/traceloop/openllmetry).
 
 ## Sending events
 
@@ -186,6 +198,67 @@ L.event("topic alignment", topic in poem)
 L.evaluate_event("excessive_wordiness", "check_wordy", {"text_input": poem})
 ```
 
+## Evaluations
+
+### Quickstart
+
+Install the package:
+
+```sh
+pip install lmnr
+```
+
+Create a file named `my_first_eval.py` with the following code:
+
+```python
+from lmnr import evaluate
+
+def write_poem(data):
+    return f"This is a good poem about {data['topic']}"
+
+def contains_poem(output, target):
+    return 1 if output in target['poem'] else 0
+
+# Evaluation data
+data = [
+    {"data": {"topic": "flowers"}, "target": {"poem": "This is a good poem about flowers"}},
+    {"data": {"topic": "cars"}, "target": {"poem": "I like cars"}},
+]
+
+evaluate(
+    data=data,
+    executor=write_poem,
+    evaluators={
+        "containsPoem": contains_poem
+    }
+)
+```
+
+Run the following commands:
+
+```sh
+export LMNR_PROJECT_API_KEY=<YOUR_PROJECT_API_KEY>  # get from Laminar project settings
+lmnr eval my_first_eval.py  # run in the virtual environment where lmnr is installed
+```
+
+Visit the URL printed in the console to see the results.
+
+### Overview
+
+Bring rigor to the development of your LLM applications with evaluations.
+
+You can run evaluations locally by providing executor (part of the logic used in your application) and evaluators (numeric scoring functions) to `evaluate` function.
+
+`evaluate` takes in the following parameters:
+- `data` – an array of `EvaluationDatapoint` objects, where each `EvaluationDatapoint` has two keys: `target` and `data`, each containing a key-value object. Alternatively, you can pass in dictionaries, and we will instantiate `EvaluationDatapoint`s with pydantic if possible
+- `executor` – the logic you want to evaluate. This function must take `data` as the first argument, and produce any output. It can be both a function or an `async` function.
+- `evaluators` – Dictionary which maps evaluator names to evaluators. Functions that take output of executor as the first argument, `target` as the second argument and produce a numeric scores. Each function can produce either a single number or `dict[str, int|float]` of scores. Each evaluator can be both a function or an `async` function.
+- `name` – optional name for the evaluation. Automatically generated if not provided.
+
+\* If you already have the outputs of executors you want to evaluate, you can specify the executor as an identity function, that takes in `data` and returns only needed value(s) from it.
+
+[Read docs](https://docs.lmnr.ai/evaluations/introduction) to learn more about evaluations.
+
 ## Laminar pipelines as prompt chain managers
 
 You can create Laminar pipelines in the UI and manage chains of LLM calls there.
@@ -220,65 +293,3 @@ PipelineRunResponse(
 )
 ```
 
-## Running offline evaluations on your data
-
-You can evaluate your code with your own data and send it to Laminar using the `Evaluation` class.
-
-Evaluation takes in the following parameters:
-- `name` – the name of your evaluation. If no such evaluation exists in the project, it will be created. Otherwise, data will be pushed to the existing evaluation
-- `data` – an array of `EvaluationDatapoint` objects, where each `EvaluationDatapoint` has two keys: `target` and `data`, each containing a key-value object. Alternatively, you can pass in dictionaries, and we will instantiate `EvaluationDatapoint`s with pydantic if possible
-- `executor` – the logic you want to evaluate. This function must take `data` as the first argument, and produce any output. *
-- `evaluators` – evaluaton logic. List of functions that take output of executor as the first argument, `target` as the second argument and produce a numeric scores. Each function can produce either a single number or `dict[str, int|float]` of scores.
-
-\* If you already have the outputs of executors you want to evaluate, you can specify the executor as an identity function, that takes in `data` and returns only needed value(s) from it.
-
-### Example
-
-```python
-from openai import AsyncOpenAI
-import asyncio
-import os
-
-openai_client = AsyncOpenAI(api_key=os.environ["OPENAI_API_KEY"])
-
-async def get_capital(data):
-    country = data["country"]
-    response = await openai_client.chat.completions.create(
-        model="gpt-4o-mini",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {
-                "role": "user",
-                "content": f"What is the capital of {country}? Just name the "
-                "city and nothing else",
-            },
-        ],
-    )
-    return response.choices[0].message.content.strip()
-
-
-# Evaluation data
-data = [
-    {"data": {"country": "Canada"}, "target": {"capital": "Ottawa"}},
-    {"data": {"country": "Germany"}, "target": {"capital": "Berlin"}},
-    {"data": {"country": "Tanzania"}, "target": {"capital": "Dodoma"}},
-]
-
-
-def evaluator_A(output, target):
-    return 1 if output == target["capital"] else 0
-
-
-# Create an Evaluation instance
-e = Evaluation(
-    name="py-evaluation-async",
-    data=data,
-    executor=get_capital,
-    evaluators=[evaluator_A],
-    project_api_key=os.environ["LMNR_PROJECT_API_KEY"],
-)
-
-# Run the evaluation
-asyncio.run(e.run())
-```
-

lmnr-0.4.12/README.md

@@ -0,0 +1,235 @@
+# Laminar Python
+
+Python SDK for [Laminar](https://www.lmnr.ai).
+
+[Laminar](https://www.lmnr.ai) is an open-source platform for engineering LLM products. Trace, evaluate, annotate, and analyze LLM data. Bring LLM applications to production with confidence.
+
+Check our [open-source repo](https://github.com/lmnr-ai/lmnr) and don't forget to star it ⭐
+
+ <a href="https://pypi.org/project/lmnr/"> ![PyPI - Version](https://img.shields.io/pypi/v/lmnr?label=lmnr&logo=pypi&logoColor=3775A9) </a>
+![PyPI - Downloads](https://img.shields.io/pypi/dm/lmnr)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/lmnr)
+
+
+## Quickstart
+
+First, install the package:
+
+```sh
+pip install lmnr
+```
+
+And then in the code
+
+```python
+from lmnr import Laminar as L
+
+L.initialize(project_api_key="<PROJECT_API_KEY>")
+```
+
+This will automatically instrument most of the LLM, Vector DB, and related
+calls with OpenTelemetry-compatible instrumentation.
+
+Note that you need to only initialize Laminar once in your application.
+
+## Instrumentation
+
+### Manual instrumentation
+
+To instrument any function in your code, we provide a simple `@observe()` decorator.
+This can be useful if you want to trace a request handler or a function which combines multiple LLM calls.
+
+```python
+import os
+from openai import OpenAI
+from lmnr import Laminar as L, Instruments
+
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])
+
+client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+def poem_writer(topic: str):
+    prompt = f"write a poem about {topic}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt},
+    ]
+
+    # OpenAI calls are still automatically instrumented
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=messages,
+    )
+    poem = response.choices[0].message.content
+
+    return poem
+
+@observe()
+def generate_poems():
+    poem1 = poem_writer(topic="laminar flow")
+    L.event("is_poem_generated", True)
+    poem2 = poem_writer(topic="turbulence")
+    L.event("is_poem_generated", True)
+    poems = f"{poem1}\n\n---\n\n{poem2}"
+    return poems
+```
+
+Also, you can use `Laminar.start_as_current_span` if you want to record a chunk of your code using `with` statement.
+
+```python
+def handle_user_request(topic: str):
+    with L.start_as_current_span(name="poem_writer", input=topic):
+        ...
+
+        poem = poem_writer(topic=topic)
+        
+        ...
+        
+        # while within the span, you can attach laminar events to it
+        L.event("is_poem_generated", True)
+
+        # Use set_span_output to record the output of the span
+        L.set_span_output(poem)
+```
+
+### Automatic instrumentation
+
+Laminar allows you to automatically instrument majority of the most popular LLM, Vector DB, database, requests, and other libraries.
+
+If you want to automatically instrument a default set of libraries, then simply do NOT pass `instruments` argument to `.initialize()`.
+See the full list of available instrumentations in the [enum](/src/lmnr/traceloop_sdk/instruments.py).
+
+If you want to automatically instrument only specific LLM, Vector DB, or other
+calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+For example, if you want to only instrument OpenAI and Anthropic, then do the following:
+
+```python
+from lmnr import Laminar as L, Instruments
+
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"], instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
+```
+
+If you want to fully disable any kind of autoinstrumentation, pass an empty set as `instruments=set()` to `.initialize()`. 
+
+Autoinstrumentations are provided by Traceloop's [OpenLLMetry](https://github.com/traceloop/openllmetry).
+
+## Sending events
+
+You can send events in two ways:
+- `.event(name, value)` – for a pre-defined event with one of possible values.
+- `.evaluate_event(name, evaluator, data)` – for an event that is evaluated by evaluator pipeline based on the data.
+
+Note that to run an evaluate event, you need to crate an evaluator pipeline and create a target version for it.
+
+Read our [docs](https://docs.lmnr.ai) to learn more about event types and how they are created and evaluated.
+
+### Example
+
+```python
+from lmnr import Laminar as L
+# ...
+poem = response.choices[0].message.content
+
+# this will register True or False value with Laminar
+L.event("topic alignment", topic in poem)
+
+# this will run the pipeline `check_wordy` with `poem` set as the value
+# of `text_input` node, and write the result as an event with name
+# "excessive_wordiness"
+L.evaluate_event("excessive_wordiness", "check_wordy", {"text_input": poem})
+```
+
+## Evaluations
+
+### Quickstart
+
+Install the package:
+
+```sh
+pip install lmnr
+```
+
+Create a file named `my_first_eval.py` with the following code:
+
+```python
+from lmnr import evaluate
+
+def write_poem(data):
+    return f"This is a good poem about {data['topic']}"
+
+def contains_poem(output, target):
+    return 1 if output in target['poem'] else 0
+
+# Evaluation data
+data = [
+    {"data": {"topic": "flowers"}, "target": {"poem": "This is a good poem about flowers"}},
+    {"data": {"topic": "cars"}, "target": {"poem": "I like cars"}},
+]
+
+evaluate(
+    data=data,
+    executor=write_poem,
+    evaluators={
+        "containsPoem": contains_poem
+    }
+)
+```
+
+Run the following commands:
+
+```sh
+export LMNR_PROJECT_API_KEY=<YOUR_PROJECT_API_KEY>  # get from Laminar project settings
+lmnr eval my_first_eval.py  # run in the virtual environment where lmnr is installed
+```
+
+Visit the URL printed in the console to see the results.
+
+### Overview
+
+Bring rigor to the development of your LLM applications with evaluations.
+
+You can run evaluations locally by providing executor (part of the logic used in your application) and evaluators (numeric scoring functions) to `evaluate` function.
+
+`evaluate` takes in the following parameters:
+- `data` – an array of `EvaluationDatapoint` objects, where each `EvaluationDatapoint` has two keys: `target` and `data`, each containing a key-value object. Alternatively, you can pass in dictionaries, and we will instantiate `EvaluationDatapoint`s with pydantic if possible
+- `executor` – the logic you want to evaluate. This function must take `data` as the first argument, and produce any output. It can be both a function or an `async` function.
+- `evaluators` – Dictionary which maps evaluator names to evaluators. Functions that take output of executor as the first argument, `target` as the second argument and produce a numeric scores. Each function can produce either a single number or `dict[str, int|float]` of scores. Each evaluator can be both a function or an `async` function.
+- `name` – optional name for the evaluation. Automatically generated if not provided.
+
+\* If you already have the outputs of executors you want to evaluate, you can specify the executor as an identity function, that takes in `data` and returns only needed value(s) from it.
+
+[Read docs](https://docs.lmnr.ai/evaluations/introduction) to learn more about evaluations.
+
+## Laminar pipelines as prompt chain managers
+
+You can create Laminar pipelines in the UI and manage chains of LLM calls there.
+
+After you are ready to use your pipeline in your code, deploy it in Laminar by selecting the target version for the pipeline.
+
+Once your pipeline target is set, you can call it from Python in just a few lines.
+
+Example use:
+
+```python
+from lmnr import Laminar as L
+
+L.initialize('<YOUR_PROJECT_API_KEY>', instruments=set())
+
+result = l.run(
+    pipeline = 'my_pipeline_name',
+    inputs = {'input_node_name': 'some_value'},
+    # all environment variables
+    env = {'OPENAI_API_KEY': 'sk-some-key'},
+)
+```
+
+Resulting in:
+
+```python
+>>> result
+PipelineRunResponse(
+    outputs={'output': {'value': [ChatMessage(role='user', content='hello')]}},
+    # useful to locate your trace
+    run_id='53b012d5-5759-48a6-a9c5-0011610e3669'
+)
+```

{lmnr-0.4.10 → lmnr-0.4.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lmnr"
-version = "0.4.10"
+version = "0.4.12"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -11,7 +11,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.10"
+version = "0.4.12"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"
@@ -33,7 +33,6 @@ opentelemetry-instrumentation-sqlalchemy = "^0.48b0"
 opentelemetry-instrumentation-urllib3 = "^0.48b0"
 opentelemetry-instrumentation-threading = "^0.48b0"
 opentelemetry-semantic-conventions-ai = "0.4.1"
-colorama = "^0.4"
 tenacity = "~=8.0"
 jinja2 = "~=3.0"
 deprecated = "~=1.0"
@@ -62,6 +61,8 @@ opentelemetry-instrumentation-weaviate = "^0.30.0"
 opentelemetry-instrumentation-alephalpha = "^0.30.0"
 opentelemetry-instrumentation-marqo = "^0.30.0"
 opentelemetry-instrumentation-groq = "^0.30.0"
+tqdm = "~=4.0"
+argparse = "~=1.0"
 
 [tool.poetry.group.dev.dependencies]
 autopep8 = "^2.2.0"
@@ -83,11 +84,8 @@ langchain-openai = "^0.1.15"
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
-[project.entry-points.console_scripts]
-lmnr = "lmnr.cli.cli:cli"
-
 [tool.poetry.scripts]
-lmnr = "lmnr.cli.cli:cli"
+lmnr = "lmnr.cli:cli"
 
 [project.optional-dependencies]
 test = ["pytest"]

{lmnr-0.4.10 → lmnr-0.4.12}/src/lmnr/__init__.py

@@ -1,4 +1,4 @@
-from .sdk.evaluations import Evaluation
+from .sdk.evaluations import evaluate
 from .sdk.laminar import Laminar
 from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput
 from .sdk.decorators import observe

lmnr-0.4.12/src/lmnr/cli.py

@@ -0,0 +1,39 @@
+from argparse import ArgumentParser
+import asyncio
+import importlib
+import os
+import sys
+
+from lmnr.sdk.evaluations import set_global_evaluation
+
+
+# TODO: Refactor this code
+async def run_evaluation(args):
+    sys.path.insert(0, os.getcwd())
+
+    with set_global_evaluation(True):
+        file = os.path.abspath(args.file)
+
+        spec = importlib.util.spec_from_file_location("run_eval", file)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+
+        from lmnr.sdk.evaluations import _evaluation
+        evaluation = _evaluation
+        await evaluation.run()
+
+
+def cli():
+    parser = ArgumentParser(
+        prog="lmnr",
+        description="CLI for Laminar",
+    )
+
+    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
+
+    parser_eval = subparsers.add_parser("eval", description="Run an evaluation")
+    parser_eval.add_argument("file", help="A file containing the evaluation to run")
+    parser_eval.set_defaults(func=run_evaluation)
+
+    parsed = parser.parse_args()
+    asyncio.run(parsed.func(parsed))