evalkeep 0.1.0__py3-none-any.whl

evalkeep/examples/__init__.py

@@ -0,0 +1,37 @@
+"""Bundled example data, shipped with the package.
+
+These live inside the package rather than beside it so that installing from
+PyPI gets them too. A quick start whose every command points at files only a
+git clone has is not a quick start.
+
+``evalkeep demo`` copies them out; :func:`root` locates them in place.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+#: Files that are not example data and should not be copied out.
+_SKIP = {"__init__.py", "__pycache__"}
+
+
+def root() -> Path:
+    """Where the bundled examples live inside the installed package."""
+    return Path(__file__).resolve().parent
+
+
+def available() -> list[str]:
+    return sorted(
+        entry.name for entry in root().iterdir() if entry.is_dir() and entry.name not in _SKIP
+    )
+
+
+def copy_to(destination: Path) -> list[Path]:
+    """Copy every bundled example into ``destination``, returning what was written."""
+    written: list[Path] = []
+    for name in available():
+        target = destination / name
+        shutil.copytree(root() / name, target, dirs_exist_ok=True)
+        written.extend(sorted(p for p in target.rglob("*") if p.is_file()))
+    return written

evalkeep/examples/langsmith/runs.jsonl

@@ -0,0 +1,18 @@
+{"id": "trace-1042-root", "trace_id": "trace-1042", "parent_run_id": null, "name": "RefundAgent", "run_type": "chain", "status": "error", "start_time": "2026-08-14T09:12:03Z", "end_time": "2026-08-14T09:12:03Z", "inputs": {"input": "Refund my latest order."}, "outputs": {"output": "I've refunded order order-A for $24.00."}, "extra": {"metadata": {"ls_model_name": "demo-model-v3", "ls_project_name": "shopping-agent"}}, "tags": ["refund"], "error": "refunded the oldest order, not the newest", "feedback": [{"key": "user_score", "score": 0, "comment": "Refunded the oldest order instead of the newest."}]}
+{"id": "trace-1042-llm", "trace_id": "trace-1042", "parent_run_id": "trace-1042-root", "name": "ChatOpenAI", "run_type": "llm", "status": "success", "start_time": "2026-08-14T09:12:03Z", "inputs": {"messages": [[{"role": "human", "content": "Refund my latest order."}]]}, "outputs": {"generations": [[{"message": {"kwargs": {"content": "I've refunded order order-A for $24.00.", "tool_calls": [{"name": "list_orders", "args": {"customer_id": "cust-77"}}, {"name": "refund_order", "args": {"order_id": "order-A"}}]}}}]]}}
+{"id": "trace-1042-tool-0", "trace_id": "trace-1042", "parent_run_id": "trace-1042-root", "name": "list_orders", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:12:03Z", "end_time": "2026-08-14T09:12:03Z", "inputs": {"customer_id": "cust-77"}, "outputs": {"result": [{"order_id": "order-A", "placed_at": "2026-06-01", "total": "24.00"}, {"order_id": "order-B", "placed_at": "2026-07-15", "total": "61.50"}, {"order_id": "order-C", "placed_at": "2026-08-12", "total": "18.99"}]}}
+{"id": "trace-1042-tool-1", "trace_id": "trace-1042", "parent_run_id": "trace-1042-root", "name": "refund_order", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:12:03Z", "end_time": "2026-08-14T09:12:03Z", "inputs": {"order_id": "order-A"}, "outputs": {"status": "refunded", "order_id": "order-A"}}
+{"id": "trace-1043-root", "trace_id": "trace-1043", "parent_run_id": null, "name": "RefundAgent", "run_type": "chain", "status": "error", "start_time": "2026-08-14T09:13:03Z", "end_time": "2026-08-14T09:13:03Z", "inputs": {"input": "Please refund the order I placed most recently."}, "outputs": {"output": "Done - order-D has been refunded."}, "extra": {"metadata": {"ls_model_name": "demo-model-v3", "ls_project_name": "shopping-agent"}}, "tags": ["refund"], "error": "expected order-C, refunded order-D"}
+{"id": "trace-1043-llm", "trace_id": "trace-1043", "parent_run_id": "trace-1043-root", "name": "ChatOpenAI", "run_type": "llm", "status": "success", "start_time": "2026-08-14T09:13:03Z", "inputs": {"messages": [[{"role": "human", "content": "Please refund the order I placed most recently."}]]}, "outputs": {"generations": [[{"message": {"kwargs": {"content": "Done - order-D has been refunded.", "tool_calls": [{"name": "list_orders", "args": {"customer_id": "cust-91"}}, {"name": "refund_order", "args": {"order_id": "order-D"}}]}}}]]}}
+{"id": "trace-1043-tool-0", "trace_id": "trace-1043", "parent_run_id": "trace-1043-root", "name": "list_orders", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:13:03Z", "end_time": "2026-08-14T09:13:03Z", "inputs": {"customer_id": "cust-91"}, "outputs": {"result": [{"order_id": "order-A", "placed_at": "2026-06-01", "total": "24.00"}, {"order_id": "order-B", "placed_at": "2026-07-15", "total": "61.50"}, {"order_id": "order-C", "placed_at": "2026-08-12", "total": "18.99"}]}}
+{"id": "trace-1043-tool-1", "trace_id": "trace-1043", "parent_run_id": "trace-1043-root", "name": "refund_order", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:13:03Z", "end_time": "2026-08-14T09:13:03Z", "inputs": {"order_id": "order-D"}, "outputs": {"status": "refunded"}}
+{"id": "trace-1051-root", "trace_id": "trace-1051", "parent_run_id": null, "name": "RefundAgent", "run_type": "chain", "status": "success", "start_time": "2026-08-14T09:14:03Z", "end_time": "2026-08-14T09:14:03Z", "inputs": {"input": "Cancel and refund order-B."}, "outputs": {"output": "I have refunded every order on your account."}, "extra": {"metadata": {"ls_model_name": "demo-model-v3", "ls_project_name": "shopping-agent"}}, "tags": ["refund"], "feedback": [{"key": "user_score", "score": 0, "comment": "Refunded three orders; I asked for one."}]}
+{"id": "trace-1051-llm", "trace_id": "trace-1051", "parent_run_id": "trace-1051-root", "name": "ChatOpenAI", "run_type": "llm", "status": "success", "start_time": "2026-08-14T09:14:03Z", "inputs": {"messages": [[{"role": "human", "content": "Cancel and refund order-B."}]]}, "outputs": {"generations": [[{"message": {"kwargs": {"content": "I have refunded every order on your account.", "tool_calls": [{"name": "refund_order", "args": {"order_id": "order-A"}}, {"name": "refund_order", "args": {"order_id": "order-B"}}, {"name": "refund_order", "args": {"order_id": "order-C"}}]}}}]]}}
+{"id": "trace-1051-tool-0", "trace_id": "trace-1051", "parent_run_id": "trace-1051-root", "name": "refund_order", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:14:03Z", "end_time": "2026-08-14T09:14:03Z", "inputs": {"order_id": "order-A"}, "outputs": {"status": "refunded"}}
+{"id": "trace-1051-tool-1", "trace_id": "trace-1051", "parent_run_id": "trace-1051-root", "name": "refund_order", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:14:03Z", "end_time": "2026-08-14T09:14:03Z", "inputs": {"order_id": "order-B"}, "outputs": {"status": "refunded"}}
+{"id": "trace-1051-tool-2", "trace_id": "trace-1051", "parent_run_id": "trace-1051-root", "name": "refund_order", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:14:03Z", "end_time": "2026-08-14T09:14:03Z", "inputs": {"order_id": "order-C"}, "outputs": {"status": "refunded"}}
+{"id": "trace-1060-root", "trace_id": "trace-1060", "parent_run_id": null, "name": "RefundAgent", "run_type": "chain", "status": "success", "start_time": "2026-08-14T09:15:03Z", "end_time": "2026-08-14T09:15:03Z", "inputs": {"input": "What is the status of order-C?"}, "outputs": {"output": "order-C shipped on 2026-08-13 and arrives Friday."}, "extra": {"metadata": {"ls_model_name": "demo-model-v3", "ls_project_name": "shopping-agent"}}, "tags": []}
+{"id": "trace-1060-llm", "trace_id": "trace-1060", "parent_run_id": "trace-1060-root", "name": "ChatOpenAI", "run_type": "llm", "status": "success", "start_time": "2026-08-14T09:15:03Z", "inputs": {"messages": [[{"role": "human", "content": "What is the status of order-C?"}]]}, "outputs": {"generations": [[{"message": {"kwargs": {"content": "order-C shipped on 2026-08-13 and arrives Friday.", "tool_calls": [{"name": "get_order", "args": {"order_id": "order-C"}}]}}}]]}}
+{"id": "trace-1060-tool-0", "trace_id": "trace-1060", "parent_run_id": "trace-1060-root", "name": "get_order", "run_type": "tool", "status": "success", "start_time": "2026-08-14T09:15:03Z", "end_time": "2026-08-14T09:15:03Z", "inputs": {"order_id": "order-C"}, "outputs": {"order_id": "order-C", "status": "shipped"}}
+{"id": "trace-1061-root", "trace_id": "trace-1061", "parent_run_id": null, "name": "RefundAgent", "run_type": "chain", "status": "success", "start_time": "2026-08-14T09:16:03Z", "end_time": "2026-08-14T09:16:03Z", "inputs": {"input": "Do you ship to Portugal?"}, "outputs": {"output": "Yes, we ship to Portugal."}, "extra": {"metadata": {"ls_model_name": "demo-model-v3", "ls_project_name": "shopping-agent"}}, "tags": []}
+{"id": "trace-1061-llm", "trace_id": "trace-1061", "parent_run_id": "trace-1061-root", "name": "ChatOpenAI", "run_type": "llm", "status": "success", "start_time": "2026-08-14T09:16:03Z", "inputs": {"messages": [[{"role": "human", "content": "Do you ship to Portugal?"}]]}, "outputs": {"generations": [[{"message": {"kwargs": {"content": "Yes, we ship to Portugal.", "tool_calls": []}}}]]}}