openrunner-sdk 2.0.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/PKG-INFO +1 -1
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/__init__.py +20 -4
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/api_client.py +16 -6
- openrunner_sdk-2.1.0/openrunner/cli.py +2364 -0
- openrunner_sdk-2.1.0/openrunner/environment.py +203 -0
- openrunner_sdk-2.1.0/openrunner/evaluation.py +681 -0
- openrunner_sdk-2.1.0/openrunner/feedback.py +136 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/git_info.py +25 -1
- openrunner_sdk-2.1.0/openrunner/guardrails.py +363 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/__init__.py +2 -0
- openrunner_sdk-2.1.0/openrunner/integration/gymnasium.py +269 -0
- openrunner_sdk-2.1.0/openrunner/integration/ignite.py +287 -0
- openrunner_sdk-2.1.0/openrunner/model.py +311 -0
- openrunner_sdk-2.1.0/openrunner/pii.py +246 -0
- openrunner_sdk-2.1.0/openrunner/prompt.py +561 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/run.py +7 -1
- openrunner_sdk-2.1.0/openrunner/scorers.py +443 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/trace.py +239 -4
- openrunner_sdk-2.1.0/openrunner/transcript_formatter.py +137 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/pyproject.toml +1 -1
- openrunner_sdk-2.1.0/tests/test_class_scorers.py +820 -0
- openrunner_sdk-2.1.0/tests/test_cli.py +543 -0
- openrunner_sdk-2.1.0/tests/test_pii.py +321 -0
- openrunner_sdk-2.1.0/tests/test_trace.py +769 -0
- openrunner_sdk-2.0.0/openrunner/cli.py +0 -1056
- openrunner_sdk-2.0.0/openrunner/evaluation.py +0 -323
- openrunner_sdk-2.0.0/openrunner/prompt.py +0 -278
- openrunner_sdk-2.0.0/openrunner/scorers.py +0 -74
- openrunner_sdk-2.0.0/tests/test_cli.py +0 -204
- openrunner_sdk-2.0.0/tests/test_trace.py +0 -385
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/.gitignore +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/=6.0 +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/=8.1 +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/README.md +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/artifact.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/buffer.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/cache.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/config.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/cost.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/dataset.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/accelerate.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/anthropic_tracer.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/catboost.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/diffusers.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/fastai.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/forced_alignment.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/gladia.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/huggingface.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/hydra.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/jax.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/keras.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/langchain.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/lightgbm.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/lightning.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/llamaindex.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/openai_finetune.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/openai_tracer.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/optuna.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/pytorch.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/sb3.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/sklearn.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/tensorflow.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/trl.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/tts.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/ultralytics.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/voice_agent.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/whisper.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/integration/xgboost.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/launch.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/media.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/migrate.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/offline.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/plot.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/query_api.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/sender.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/settings.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/summary.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/sweep.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/system_metrics.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/tensorboard.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/wal.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/wandb_compat/__init__.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/openrunner/wandb_compat/_shim.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/__init__.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/conftest.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_alert.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_aliases.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_api_client.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_artifact.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_buffer.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_cache.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_config.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_evaluation.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_finish.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_git_info.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_init.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_fastai.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_huggingface.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_keras.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_langchain.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_lightning.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_pytorch.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_sklearn.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_integration_xgboost.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_launch.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_log.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_log_code.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_media.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_migrate.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_offline.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_offline_sync.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_plot.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_query_api.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_resume.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_sdk_features.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_sender.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_summary.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_sweep.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_system_metrics.py +0 -0
- {openrunner_sdk-2.0.0 → openrunner_sdk-2.1.0}/tests/test_wandb_compat.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openrunner-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
|
|
5
5
|
Project-URL: Homepage, https://github.com/jqueguiner/openrunner
|
|
6
6
|
Project-URL: Repository, https://github.com/jqueguiner/openrunner
|
|
@@ -38,11 +38,16 @@ Public API:
|
|
|
38
38
|
openrunner.trace -> Decorator for LLM call tracing (supports sample_rate)
|
|
39
39
|
openrunner.trace.patch_openai -> Auto-trace OpenAI chat completions
|
|
40
40
|
openrunner.Prompt -> Versioned prompt management (get/render/publish)
|
|
41
|
+
openrunner.StringPrompt -> Client-side string template with format()
|
|
42
|
+
openrunner.MessagesPrompt -> Multi-turn chat prompt template
|
|
43
|
+
openrunner.Model -> Versioned model with predict/save/load/serve
|
|
41
44
|
openrunner.Dataset -> Versioned dataset for evaluations
|
|
42
45
|
openrunner.add_cost -> Register custom LLM cost per model
|
|
43
46
|
openrunner.evaluate -> Run LLM evaluation against a dataset
|
|
44
47
|
openrunner.scorer -> Decorator to mark a function as an eval scorer
|
|
45
|
-
openrunner.
|
|
48
|
+
openrunner.Scorer -> Base class for class-based evaluation scorers
|
|
49
|
+
openrunner.EvaluationLogger -> Log evaluation results piecemeal
|
|
50
|
+
openrunner.scorers -> Built-in scorers (exact_match, contains, LLM-as-judge)
|
|
46
51
|
openrunner.plot.line -> Create a line chart from a Table
|
|
47
52
|
openrunner.plot.scatter -> Create a scatter chart from a Table
|
|
48
53
|
openrunner.plot.bar -> Create a bar chart from a Table
|
|
@@ -61,7 +66,8 @@ from typing import Any
|
|
|
61
66
|
|
|
62
67
|
from openrunner.artifact import Artifact
|
|
63
68
|
from openrunner.config import Config
|
|
64
|
-
from openrunner.
|
|
69
|
+
from openrunner.model import Model
|
|
70
|
+
from openrunner.prompt import MessagesPrompt, Prompt, StringPrompt
|
|
65
71
|
from openrunner.launch import LaunchJob, launch, from_run as _launch_from_run
|
|
66
72
|
from openrunner.media import (
|
|
67
73
|
Audio,
|
|
@@ -90,8 +96,18 @@ from openrunner.run import Run
|
|
|
90
96
|
from openrunner.settings import SDKSettings
|
|
91
97
|
from openrunner.summary import Summary
|
|
92
98
|
from openrunner.sweep import agent, sweep
|
|
93
|
-
from openrunner.evaluation import evaluate, scorer
|
|
94
|
-
from openrunner.
|
|
99
|
+
from openrunner.evaluation import EvaluationLogger, Scorer, evaluate, scorer
|
|
100
|
+
from openrunner.guardrails import (
|
|
101
|
+
GuardrailCheckResult,
|
|
102
|
+
GuardrailResult,
|
|
103
|
+
add_guardrail,
|
|
104
|
+
check_guardrails,
|
|
105
|
+
clear_guardrails,
|
|
106
|
+
remove_guardrail,
|
|
107
|
+
)
|
|
108
|
+
from openrunner.feedback import add_feedback, get_feedback
|
|
109
|
+
from openrunner.pii import configure_pii, redact
|
|
110
|
+
from openrunner.trace import trace, patch_openai as _patch_openai, set_thread_id, set_session_id
|
|
95
111
|
import openrunner.scorers as scorers
|
|
96
112
|
import openrunner.plot as plot
|
|
97
113
|
|
|
@@ -594,19 +594,29 @@ class APIClient:
|
|
|
594
594
|
return []
|
|
595
595
|
|
|
596
596
|
def upload_file_to_presigned_url(
|
|
597
|
-
self, presigned_url: str, file_path: str
|
|
597
|
+
self, presigned_url: str, file_path: str, run_id: str = ""
|
|
598
598
|
) -> bool:
|
|
599
599
|
"""PUT raw file bytes to a presigned URL.
|
|
600
600
|
|
|
601
|
-
|
|
602
|
-
|
|
601
|
+
Falls back to API proxy upload if presigned URL is unreachable
|
|
602
|
+
(e.g., MinIO not exposed publicly).
|
|
603
603
|
"""
|
|
604
604
|
try:
|
|
605
605
|
with open(file_path, "rb") as f:
|
|
606
606
|
data = f.read()
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
607
|
+
# Try presigned URL first
|
|
608
|
+
try:
|
|
609
|
+
resp = httpx.put(presigned_url, content=data, timeout=300.0)
|
|
610
|
+
resp.raise_for_status()
|
|
611
|
+
return True
|
|
612
|
+
except (httpx.ConnectError, httpx.ConnectTimeout):
|
|
613
|
+
# Presigned URL unreachable — use API proxy
|
|
614
|
+
if run_id:
|
|
615
|
+
import os
|
|
616
|
+
fname = os.path.basename(file_path)
|
|
617
|
+
proxy_resp = self._request("PUT", f"/runs/{run_id}/files/{fname}", content=data)
|
|
618
|
+
return proxy_resp.status_code == 200
|
|
619
|
+
return False
|
|
610
620
|
except Exception as e:
|
|
611
621
|
logger.warning("upload_file_to_presigned_url failed: %s", e)
|
|
612
622
|
return False
|