runport 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. runport/__init__.py +11 -0
  2. runport/_web/apple-touch-icon.png +0 -0
  3. runport/_web/assets/index-CJJA1jJ3.js +170 -0
  4. runport/_web/assets/index-DDIDJwyV.css +1 -0
  5. runport/_web/favicon-32.png +0 -0
  6. runport/_web/favicon-512.png +0 -0
  7. runport/_web/index.html +20 -0
  8. runport/_web/runport-emblem.png +0 -0
  9. runport/api/__init__.py +5 -0
  10. runport/api/app.py +161 -0
  11. runport/api/routes/budget.py +75 -0
  12. runport/api/routes/datasets.py +130 -0
  13. runport/api/routes/events.py +47 -0
  14. runport/api/routes/models.py +64 -0
  15. runport/api/routes/plan.py +45 -0
  16. runport/api/routes/reports.py +243 -0
  17. runport/api/routes/runs.py +199 -0
  18. runport/api/routes/setup.py +88 -0
  19. runport/api/routes/targets.py +215 -0
  20. runport/catalog/__init__.py +6 -0
  21. runport/catalog/models.py +23 -0
  22. runport/catalog/openrouter.py +171 -0
  23. runport/classification/__init__.py +65 -0
  24. runport/classification/contracts.py +102 -0
  25. runport/classification/metrics.py +236 -0
  26. runport/classification/records.py +157 -0
  27. runport/classification/runner.py +145 -0
  28. runport/classification/splits.py +79 -0
  29. runport/classification/threshold.py +200 -0
  30. runport/classification/validators.py +49 -0
  31. runport/cli/__init__.py +6 -0
  32. runport/cli/commands/__init__.py +5 -0
  33. runport/cli/commands/benchmarks.py +677 -0
  34. runport/cli/commands/budget.py +221 -0
  35. runport/cli/commands/datasets.py +159 -0
  36. runport/cli/commands/init.py +49 -0
  37. runport/cli/commands/models.py +40 -0
  38. runport/cli/commands/plans.py +53 -0
  39. runport/cli/commands/traces.py +180 -0
  40. runport/cli/commands/ui.py +96 -0
  41. runport/cli/main.py +169 -0
  42. runport/connectors/__init__.py +29 -0
  43. runport/connectors/dify.py +422 -0
  44. runport/connectors/flowise.py +50 -0
  45. runport/connectors/http_workflow.py +205 -0
  46. runport/connectors/langflow.py +52 -0
  47. runport/connectors/langgraph.py +48 -0
  48. runport/connectors/mcp.py +920 -0
  49. runport/connectors/n8n.py +44 -0
  50. runport/connectors/ollama.py +156 -0
  51. runport/connectors/openai_compat.py +176 -0
  52. runport/connectors/openrouter.py +110 -0
  53. runport/connectors/python_target.py +167 -0
  54. runport/connectors/registry.py +530 -0
  55. runport/core/__init__.py +1 -0
  56. runport/core/approvers.py +177 -0
  57. runport/core/budget.py +238 -0
  58. runport/core/budget_policy.py +134 -0
  59. runport/core/config.py +177 -0
  60. runport/core/live_guard.py +80 -0
  61. runport/core/plugins.py +43 -0
  62. runport/core/security.py +137 -0
  63. runport/core/workspace.py +359 -0
  64. runport/dataset/__init__.py +18 -0
  65. runport/dataset/leakage.py +150 -0
  66. runport/dataset/manager.py +178 -0
  67. runport/datasets/__init__.py +1 -0
  68. runport/datasets/redaction.py +71 -0
  69. runport/evaluation/__init__.py +10 -0
  70. runport/evaluation/evaluator.py +77 -0
  71. runport/evaluation/replay_scoring.py +97 -0
  72. runport/evaluation/runner.py +128 -0
  73. runport/mcp_server/__init__.py +1 -0
  74. runport/mcp_server/server.py +444 -0
  75. runport/models/__init__.py +1 -0
  76. runport/models/catalog.py +115 -0
  77. runport/models/registry.py +354 -0
  78. runport/observability/__init__.py +1 -0
  79. runport/observability/aggregate.py +262 -0
  80. runport/observability/analysis.py +478 -0
  81. runport/observability/annotation_templates.py +35 -0
  82. runport/observability/budget_audit.py +157 -0
  83. runport/observability/bundle_verify.py +161 -0
  84. runport/observability/events.py +132 -0
  85. runport/observability/export.py +1012 -0
  86. runport/observability/labels.py +84 -0
  87. runport/observability/monitor.py +160 -0
  88. runport/providers/__init__.py +31 -0
  89. runport/providers/base.py +195 -0
  90. runport/providers/http.py +133 -0
  91. runport/providers/openrouter.py +243 -0
  92. runport/rag/__init__.py +23 -0
  93. runport/rag/evaluator.py +79 -0
  94. runport/rag/models.py +99 -0
  95. runport/rag/retriever.py +215 -0
  96. runport/rag/runner.py +311 -0
  97. runport/rag/target.py +76 -0
  98. runport/reports/__init__.py +5 -0
  99. runport/reports/generator.py +207 -0
  100. runport/runs/__init__.py +5 -0
  101. runport/runs/executor.py +352 -0
  102. runport/runs/planner.py +90 -0
  103. runport/runs/replay.py +177 -0
  104. runport/schemas/__init__.py +23 -0
  105. runport/schemas/evaluation.py +114 -0
  106. runport/schemas/execution.py +23 -0
  107. runport/schemas/scoring.py +33 -0
  108. runport/scorers/__init__.py +39 -0
  109. runport/scorers/base.py +41 -0
  110. runport/scorers/classification.py +224 -0
  111. runport/scorers/extraction.py +100 -0
  112. runport/scorers/generation.py +90 -0
  113. runport/scorers/llm_judge.py +143 -0
  114. runport/scorers/match.py +111 -0
  115. runport/scorers/performance.py +136 -0
  116. runport/scorers/stats.py +96 -0
  117. runport/sdk.py +149 -0
  118. runport/services/__init__.py +1 -0
  119. runport/services/approvals.py +174 -0
  120. runport/services/execution.py +322 -0
  121. runport/storage/__init__.py +1 -0
  122. runport/storage/db.py +635 -0
  123. runport/targets/__init__.py +25 -0
  124. runport/targets/base.py +185 -0
  125. runport/traces/__init__.py +23 -0
  126. runport/traces/analyzer.py +276 -0
  127. runport/traces/importer.py +252 -0
  128. runport/traces/models.py +76 -0
  129. runport/traces/reporter.py +98 -0
  130. runport/tui/__init__.py +5 -0
  131. runport/tui/app.py +1000 -0
  132. runport/tui/banner.py +22 -0
  133. runport/tui/components.py +428 -0
  134. runport/tui/health.py +184 -0
  135. runport/tui/input.py +111 -0
  136. runport/tui/live_views.py +991 -0
  137. runport/tui/logo_ascii.py +79 -0
  138. runport/tui/onboarding.py +204 -0
  139. runport/tui/screens.py +1592 -0
  140. runport/tui/telemetry.py +178 -0
  141. runport/tui/theme.py +131 -0
  142. runport/tui/welcome.py +376 -0
  143. runport/workloads/__init__.py +34 -0
  144. runport/workloads/base.py +29 -0
  145. runport/workloads/classification.py +36 -0
  146. runport/workloads/extraction.py +47 -0
  147. runport/workloads/generation.py +32 -0
  148. runport/workloads/rag.py +32 -0
  149. runport-0.1.0.dist-info/METADATA +342 -0
  150. runport-0.1.0.dist-info/RECORD +154 -0
  151. runport-0.1.0.dist-info/WHEEL +4 -0
  152. runport-0.1.0.dist-info/entry_points.txt +2 -0
  153. runport-0.1.0.dist-info/licenses/LICENSE +202 -0
  154. runport-0.1.0.dist-info/licenses/NOTICE +6 -0
runport/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """Runport: Universal open-source LLM, prompt, and workflow benchmarking platform."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+
6
+ def __getattr__(name: str): # lazy, so importing runport stays cheap
7
+ if name in ("benchmark", "plan", "execute", "approve_interactively", "approve_with_ci_token"):
8
+ from runport import sdk
9
+
10
+ return getattr(sdk, name)
11
+ raise AttributeError(name)
runport/_web/apple-touch-icon.png ADDED
Binary file