openrunner-sdk 2.0.0__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/PKG-INFO +1 -1
  2. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/__init__.py +20 -4
  3. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/api_client.py +16 -6
  4. openrunner_sdk-2.2.0/openrunner/cli.py +2364 -0
  5. openrunner_sdk-2.2.0/openrunner/environment.py +203 -0
  6. openrunner_sdk-2.2.0/openrunner/evaluation.py +681 -0
  7. openrunner_sdk-2.2.0/openrunner/feedback.py +136 -0
  8. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/git_info.py +25 -1
  9. openrunner_sdk-2.2.0/openrunner/guardrails.py +363 -0
  10. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/__init__.py +2 -0
  11. openrunner_sdk-2.2.0/openrunner/integration/gymnasium.py +269 -0
  12. openrunner_sdk-2.2.0/openrunner/integration/ignite.py +287 -0
  13. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/media.py +91 -1
  14. openrunner_sdk-2.2.0/openrunner/model.py +311 -0
  15. openrunner_sdk-2.2.0/openrunner/pii.py +246 -0
  16. openrunner_sdk-2.2.0/openrunner/prompt.py +561 -0
  17. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/run.py +7 -1
  18. openrunner_sdk-2.2.0/openrunner/scorers.py +443 -0
  19. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/trace.py +239 -4
  20. openrunner_sdk-2.2.0/openrunner/transcript_formatter.py +137 -0
  21. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/pyproject.toml +1 -1
  22. openrunner_sdk-2.2.0/tests/test_class_scorers.py +820 -0
  23. openrunner_sdk-2.2.0/tests/test_cli.py +543 -0
  24. openrunner_sdk-2.2.0/tests/test_pii.py +321 -0
  25. openrunner_sdk-2.2.0/tests/test_trace.py +769 -0
  26. openrunner_sdk-2.0.0/openrunner/cli.py +0 -1056
  27. openrunner_sdk-2.0.0/openrunner/evaluation.py +0 -323
  28. openrunner_sdk-2.0.0/openrunner/prompt.py +0 -278
  29. openrunner_sdk-2.0.0/openrunner/scorers.py +0 -74
  30. openrunner_sdk-2.0.0/tests/test_cli.py +0 -204
  31. openrunner_sdk-2.0.0/tests/test_trace.py +0 -385
  32. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/.gitignore +0 -0
  33. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/=6.0 +0 -0
  34. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/=8.1 +0 -0
  35. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/README.md +0 -0
  36. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/artifact.py +0 -0
  37. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/buffer.py +0 -0
  38. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/cache.py +0 -0
  39. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/config.py +0 -0
  40. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/cost.py +0 -0
  41. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/dataset.py +0 -0
  42. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/accelerate.py +0 -0
  43. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/anthropic_tracer.py +0 -0
  44. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/catboost.py +0 -0
  45. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/diffusers.py +0 -0
  46. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/fastai.py +0 -0
  47. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/forced_alignment.py +0 -0
  48. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/gladia.py +0 -0
  49. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/huggingface.py +0 -0
  50. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/hydra.py +0 -0
  51. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/jax.py +0 -0
  52. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/keras.py +0 -0
  53. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/langchain.py +0 -0
  54. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/lightgbm.py +0 -0
  55. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/lightning.py +0 -0
  56. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/llamaindex.py +0 -0
  57. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/openai_finetune.py +0 -0
  58. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/openai_tracer.py +0 -0
  59. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/optuna.py +0 -0
  60. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/pytorch.py +0 -0
  61. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/sb3.py +0 -0
  62. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/sklearn.py +0 -0
  63. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/tensorflow.py +0 -0
  64. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/trl.py +0 -0
  65. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/tts.py +0 -0
  66. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/ultralytics.py +0 -0
  67. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/voice_agent.py +0 -0
  68. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/whisper.py +0 -0
  69. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/integration/xgboost.py +0 -0
  70. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/launch.py +0 -0
  71. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/migrate.py +0 -0
  72. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/offline.py +0 -0
  73. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/plot.py +0 -0
  74. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/query_api.py +0 -0
  75. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/sender.py +0 -0
  76. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/settings.py +0 -0
  77. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/summary.py +0 -0
  78. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/sweep.py +0 -0
  79. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/system_metrics.py +0 -0
  80. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/tensorboard.py +0 -0
  81. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/wal.py +0 -0
  82. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/wandb_compat/__init__.py +0 -0
  83. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/openrunner/wandb_compat/_shim.py +0 -0
  84. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/__init__.py +0 -0
  85. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/conftest.py +0 -0
  86. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_alert.py +0 -0
  87. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_aliases.py +0 -0
  88. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_api_client.py +0 -0
  89. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_artifact.py +0 -0
  90. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_buffer.py +0 -0
  91. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_cache.py +0 -0
  92. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_config.py +0 -0
  93. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_evaluation.py +0 -0
  94. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_finish.py +0 -0
  95. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_git_info.py +0 -0
  96. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_init.py +0 -0
  97. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_fastai.py +0 -0
  98. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_huggingface.py +0 -0
  99. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_keras.py +0 -0
  100. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_langchain.py +0 -0
  101. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_lightning.py +0 -0
  102. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_pytorch.py +0 -0
  103. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_sklearn.py +0 -0
  104. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_integration_xgboost.py +0 -0
  105. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_launch.py +0 -0
  106. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_log.py +0 -0
  107. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_log_code.py +0 -0
  108. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_media.py +0 -0
  109. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_migrate.py +0 -0
  110. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_offline.py +0 -0
  111. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_offline_sync.py +0 -0
  112. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_plot.py +0 -0
  113. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_query_api.py +0 -0
  114. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_resume.py +0 -0
  115. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_sdk_features.py +0 -0
  116. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_sender.py +0 -0
  117. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_summary.py +0 -0
  118. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_sweep.py +0 -0
  119. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_system_metrics.py +0 -0
  120. {openrunner_sdk-2.0.0 → openrunner_sdk-2.2.0}/tests/test_wandb_compat.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: openrunner-sdk
3
- Version: 2.0.0
3
+ Version: 2.2.0
4
4
  Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
5
5
  Project-URL: Homepage, https://github.com/jqueguiner/openrunner
6
6
  Project-URL: Repository, https://github.com/jqueguiner/openrunner
@@ -38,11 +38,16 @@ Public API:
38
38
  openrunner.trace -> Decorator for LLM call tracing (supports sample_rate)
39
39
  openrunner.trace.patch_openai -> Auto-trace OpenAI chat completions
40
40
  openrunner.Prompt -> Versioned prompt management (get/render/publish)
41
+ openrunner.StringPrompt -> Client-side string template with format()
42
+ openrunner.MessagesPrompt -> Multi-turn chat prompt template
43
+ openrunner.Model -> Versioned model with predict/save/load/serve
41
44
  openrunner.Dataset -> Versioned dataset for evaluations
42
45
  openrunner.add_cost -> Register custom LLM cost per model
43
46
  openrunner.evaluate -> Run LLM evaluation against a dataset
44
47
  openrunner.scorer -> Decorator to mark a function as an eval scorer
45
- openrunner.scorers -> Built-in scorers (exact_match, contains, etc.)
48
+ openrunner.Scorer -> Base class for class-based evaluation scorers
49
+ openrunner.EvaluationLogger -> Log evaluation results piecemeal
50
+ openrunner.scorers -> Built-in scorers (exact_match, contains, LLM-as-judge)
46
51
  openrunner.plot.line -> Create a line chart from a Table
47
52
  openrunner.plot.scatter -> Create a scatter chart from a Table
48
53
  openrunner.plot.bar -> Create a bar chart from a Table
@@ -61,7 +66,8 @@ from typing import Any
61
66
 
62
67
  from openrunner.artifact import Artifact
63
68
  from openrunner.config import Config
64
- from openrunner.prompt import Prompt
69
+ from openrunner.model import Model
70
+ from openrunner.prompt import MessagesPrompt, Prompt, StringPrompt
65
71
  from openrunner.launch import LaunchJob, launch, from_run as _launch_from_run
66
72
  from openrunner.media import (
67
73
  Audio,
@@ -90,8 +96,18 @@ from openrunner.run import Run
90
96
  from openrunner.settings import SDKSettings
91
97
  from openrunner.summary import Summary
92
98
  from openrunner.sweep import agent, sweep
93
- from openrunner.evaluation import evaluate, scorer
94
- from openrunner.trace import trace, patch_openai as _patch_openai
99
+ from openrunner.evaluation import EvaluationLogger, Scorer, evaluate, scorer
100
+ from openrunner.guardrails import (
101
+ GuardrailCheckResult,
102
+ GuardrailResult,
103
+ add_guardrail,
104
+ check_guardrails,
105
+ clear_guardrails,
106
+ remove_guardrail,
107
+ )
108
+ from openrunner.feedback import add_feedback, get_feedback
109
+ from openrunner.pii import configure_pii, redact
110
+ from openrunner.trace import trace, patch_openai as _patch_openai, set_thread_id, set_session_id
95
111
  import openrunner.scorers as scorers
96
112
  import openrunner.plot as plot
97
113
 
@@ -594,19 +594,29 @@ class APIClient:
594
594
  return []
595
595
 
596
596
  def upload_file_to_presigned_url(
597
- self, presigned_url: str, file_path: str
597
+ self, presigned_url: str, file_path: str, run_id: str = ""
598
598
  ) -> bool:
599
599
  """PUT raw file bytes to a presigned URL.
600
600
 
601
- Uses httpx directly (not self._client) because presigned URLs
602
- are absolute and should not have a base_url prefix.
601
+ Falls back to API proxy upload if presigned URL is unreachable
602
+ (e.g., MinIO not exposed publicly).
603
603
  """
604
604
  try:
605
605
  with open(file_path, "rb") as f:
606
606
  data = f.read()
607
- resp = httpx.put(presigned_url, content=data, timeout=300.0)
608
- resp.raise_for_status()
609
- return True
607
+ # Try presigned URL first
608
+ try:
609
+ resp = httpx.put(presigned_url, content=data, timeout=300.0)
610
+ resp.raise_for_status()
611
+ return True
612
+ except (httpx.ConnectError, httpx.ConnectTimeout):
613
+ # Presigned URL unreachable — use API proxy
614
+ if run_id:
615
+ import os
616
+ fname = os.path.basename(file_path)
617
+ proxy_resp = self._request("PUT", f"/runs/{run_id}/files/{fname}", content=data)
618
+ return proxy_resp.status_code == 200
619
+ return False
610
620
  except Exception as e:
611
621
  logger.warning("upload_file_to_presigned_url failed: %s", e)
612
622
  return False