opengrade-ai-server 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (391) hide show
  1. opengrade_ai_server-0.1.0/.dockerignore +8 -0
  2. opengrade_ai_server-0.1.0/.env.example +36 -0
  3. opengrade_ai_server-0.1.0/.gitignore +42 -0
  4. opengrade_ai_server-0.1.0/Dockerfile +36 -0
  5. opengrade_ai_server-0.1.0/PKG-INFO +55 -0
  6. opengrade_ai_server-0.1.0/README.md +25 -0
  7. opengrade_ai_server-0.1.0/alembic/env.py +65 -0
  8. opengrade_ai_server-0.1.0/alembic/script.py.mako +18 -0
  9. opengrade_ai_server-0.1.0/alembic/versions/0001_create_projects.py +39 -0
  10. opengrade_ai_server-0.1.0/alembic/versions/0002_create_traces_spans.py +102 -0
  11. opengrade_ai_server-0.1.0/alembic/versions/0003_create_evaluation_tables.py +190 -0
  12. opengrade_ai_server-0.1.0/alembic/versions/0004_rename_experiment_application_version.py +60 -0
  13. opengrade_ai_server-0.1.0/alembic/versions/0005_datasets_task_type.py +31 -0
  14. opengrade_ai_server-0.1.0/alembic/versions/0006_experiment_item_outputs.py +64 -0
  15. opengrade_ai_server-0.1.0/alembic/versions/0007_app_configs.py +125 -0
  16. opengrade_ai_server-0.1.0/alembic/versions/0008_metrics_packs.py +48 -0
  17. opengrade_ai_server-0.1.0/alembic/versions/0009_metrics_sets.py +276 -0
  18. opengrade_ai_server-0.1.0/alembic/versions/0010_live_interactions.py +158 -0
  19. opengrade_ai_server-0.1.0/alembic/versions/0011_judge_claim_cache.py +63 -0
  20. opengrade_ai_server-0.1.0/alembic/versions/0012_live_score_reviews.py +54 -0
  21. opengrade_ai_server-0.1.0/alembic/versions/0013_experiment_metrics.py +43 -0
  22. opengrade_ai_server-0.1.0/alembic/versions/0014_trace_scores.py +69 -0
  23. opengrade_ai_server-0.1.0/alembic/versions/0015_drop_live_reviews.py +54 -0
  24. opengrade_ai_server-0.1.0/alembic.ini +42 -0
  25. opengrade_ai_server-0.1.0/docker-entrypoint.sh +8 -0
  26. opengrade_ai_server-0.1.0/hatch_build.py +20 -0
  27. opengrade_ai_server-0.1.0/pyproject.toml +105 -0
  28. opengrade_ai_server-0.1.0/src/opengrade_server/__init__.py +9 -0
  29. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/404/index.html +1 -0
  30. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/404.html +1 -0
  31. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/__next.__PAGE__.txt +24 -0
  32. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/__next._full.txt +19 -0
  33. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/__next._tree.txt +2 -0
  34. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/NGE_1bmdWAvErxLGmpEVy/_buildManifest.js +1 -0
  35. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/NGE_1bmdWAvErxLGmpEVy/_ssgManifest.js +1 -0
  36. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/1076-757ffd06fe9a315d.js +1 -0
  37. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/2188-5148a2956b7ef5af.js +1 -0
  38. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/279.f5b21c0c9baed769.js +1 -0
  39. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/3118.2fb6b163dbbe8310.js +1 -0
  40. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/3526-ec617d2c01bdba59.js +2 -0
  41. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/353-746b0c9917d28daa.js +14 -0
  42. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/3794-6c302691d4e00ed6.js +52 -0
  43. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/4005-118423359becaee1.js +1 -0
  44. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/4321.0b1eb9969f4ffe4b.js +1 -0
  45. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/4461.a254f001c7a64e29.js +1 -0
  46. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/4531-d3730f261329c1a9.js +1 -0
  47. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/4726-f5e380cd1e7ec451.js +1 -0
  48. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/4bd1b696-8a4ab4fdf0ae305a.js +1 -0
  49. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/5294.68b976daed5b0099.js +1 -0
  50. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/5803-c7fa27797e77cc31.js +1 -0
  51. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/7138-ecbc18bdac6dd064.js +12 -0
  52. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/7332.21ce747727554703.js +1 -0
  53. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/7769-f5709968e5d7d5e5.js +1 -0
  54. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/7854-2105a66d0d990217.js +1 -0
  55. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/7915-c84266b50d37948c.js +1 -0
  56. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/app-configs/[name]/page-3063ba5236ab3043.js +8 -0
  57. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/app-configs/page-625d7fb7e67c48f4.js +1 -0
  58. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/datasets/[datasetId]/page-7c439b4c2b469bff.js +4 -0
  59. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/datasets/page-f44a75ca8886701c.js +1 -0
  60. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/experiments/[experimentId]/compare/page-c4b948ffe06dd8c2.js +1 -0
  61. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/experiments/[experimentId]/page-0be47a36b7550e8f.js +24 -0
  62. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/experiments/compare/page-5f3d24c5a92c275e.js +1 -0
  63. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/experiments/page-b746f85e873a4947.js +1 -0
  64. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/layout-3cccfff334d7424a.js +1 -0
  65. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/live-runs/[interactionId]/page-dcaf09d79defbe3b.js +1 -0
  66. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/live-runs/page-e7db3f780f4e1d98.js +1 -0
  67. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/metrics/[metricsSetId]/page-e8ea81d07748f619.js +1 -0
  68. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/metrics/page-549548cd9cc8e9c7.js +1 -0
  69. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/monitoring/page-05f66912146a6a11.js +1 -0
  70. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/overview/page-f42133bc41cd427f.js +1 -0
  71. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/projects/page-e2db15da577175bb.js +1 -0
  72. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/release/page-19948b40341c25cc.js +1 -0
  73. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/traces/[traceId]/page-ce35f396bec5e9db.js +1 -0
  74. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/(app)/traces/page-d738f698f65b1805.js +1 -0
  75. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/_global-error/page-26287a6e4941226d.js +1 -0
  76. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/_not-found/page-004ce51273a8cf00.js +1 -0
  77. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/layout-57f357eed10f315e.js +1 -0
  78. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/app/page-2bb65b6f0c5bf011.js +1 -0
  79. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/framework-b33a8f5d57cd8d87.js +1 -0
  80. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/main-app-b6e651b9a20fffce.js +1 -0
  81. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/main-b914ac27a970e263.js +5 -0
  82. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/polyfills-42372ed130431b0a.js +1 -0
  83. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/chunks/webpack-c3faa3f009157ce2.js +1 -0
  84. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_next/static/css/f84620e20960c237.css +1 -0
  85. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_not-found/__next._full.txt +17 -0
  86. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_not-found/__next._not-found.__PAGE__.txt +23 -0
  87. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_not-found/__next._tree.txt +2 -0
  88. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_not-found/index.html +1 -0
  89. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/_not-found/index.txt +17 -0
  90. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/_/__next.!KGFwcCk.app-configs.$d$name.__PAGE__.txt +27 -0
  91. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/_/__next._full.txt +19 -0
  92. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/_/__next._tree.txt +2 -0
  93. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/_/index.html +1 -0
  94. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/_/index.txt +19 -0
  95. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/__next.!KGFwcCk.app-configs.__PAGE__.txt +29 -0
  96. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/__next._full.txt +22 -0
  97. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/__next._tree.txt +2 -0
  98. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/index.html +1 -0
  99. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/app-configs/index.txt +22 -0
  100. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/_/__next.!KGFwcCk.datasets.$d$datasetId.__PAGE__.txt +27 -0
  101. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/_/__next._full.txt +19 -0
  102. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/_/__next._tree.txt +2 -0
  103. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/_/index.html +1 -0
  104. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/_/index.txt +19 -0
  105. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/__next.!KGFwcCk.datasets.__PAGE__.txt +29 -0
  106. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/__next._full.txt +22 -0
  107. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/__next._tree.txt +2 -0
  108. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/index.html +1 -0
  109. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/datasets/index.txt +22 -0
  110. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/__next.!KGFwcCk.experiments.$d$experimentId.__PAGE__.txt +27 -0
  111. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/__next._full.txt +19 -0
  112. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/__next._tree.txt +2 -0
  113. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/compare/__next.!KGFwcCk.experiments.$d$experimentId.compare.__PAGE__.txt +28 -0
  114. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/compare/__next._full.txt +19 -0
  115. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/compare/__next._tree.txt +2 -0
  116. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/compare/index.html +1 -0
  117. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/compare/index.txt +19 -0
  118. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/index.html +1 -0
  119. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/_/index.txt +19 -0
  120. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/__next.!KGFwcCk.experiments.__PAGE__.txt +29 -0
  121. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/__next._full.txt +22 -0
  122. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/__next._tree.txt +2 -0
  123. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/compare/__next.!KGFwcCk.experiments.compare.__PAGE__.txt +30 -0
  124. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/compare/__next._full.txt +22 -0
  125. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/compare/__next._tree.txt +2 -0
  126. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/compare/index.html +1 -0
  127. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/compare/index.txt +22 -0
  128. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/index.html +1 -0
  129. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/experiments/index.txt +22 -0
  130. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/favicon.ico +0 -0
  131. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/file.svg +1 -0
  132. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/globe.svg +1 -0
  133. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/index.html +1 -0
  134. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/index.txt +19 -0
  135. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/_/__next.!KGFwcCk.live-runs.$d$interactionId.__PAGE__.txt +27 -0
  136. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/_/__next._full.txt +19 -0
  137. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/_/__next._tree.txt +2 -0
  138. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/_/index.html +1 -0
  139. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/_/index.txt +19 -0
  140. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/__next.!KGFwcCk.live-runs.__PAGE__.txt +29 -0
  141. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/__next._full.txt +22 -0
  142. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/__next._tree.txt +2 -0
  143. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/index.html +1 -0
  144. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/live-runs/index.txt +22 -0
  145. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/_/__next.!KGFwcCk.metrics.$d$metricsSetId.__PAGE__.txt +27 -0
  146. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/_/__next._full.txt +19 -0
  147. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/_/__next._tree.txt +2 -0
  148. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/_/index.html +1 -0
  149. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/_/index.txt +19 -0
  150. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/__next.!KGFwcCk.metrics.__PAGE__.txt +29 -0
  151. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/__next._full.txt +22 -0
  152. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/__next._tree.txt +2 -0
  153. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/index.html +1 -0
  154. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/metrics/index.txt +22 -0
  155. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/monitoring/__next.!KGFwcCk.monitoring.__PAGE__.txt +29 -0
  156. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/monitoring/__next._full.txt +22 -0
  157. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/monitoring/__next._tree.txt +2 -0
  158. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/monitoring/index.html +1 -0
  159. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/monitoring/index.txt +22 -0
  160. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/next.svg +1 -0
  161. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/overview/__next.!KGFwcCk.overview.__PAGE__.txt +29 -0
  162. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/overview/__next._full.txt +22 -0
  163. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/overview/__next._tree.txt +2 -0
  164. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/overview/index.html +1 -0
  165. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/overview/index.txt +22 -0
  166. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/projects/__next.!KGFwcCk.projects.__PAGE__.txt +29 -0
  167. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/projects/__next._full.txt +22 -0
  168. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/projects/__next._tree.txt +2 -0
  169. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/projects/index.html +1 -0
  170. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/projects/index.txt +22 -0
  171. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/release/__next.!KGFwcCk.release.__PAGE__.txt +29 -0
  172. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/release/__next._full.txt +22 -0
  173. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/release/__next._tree.txt +2 -0
  174. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/release/index.html +1 -0
  175. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/release/index.txt +22 -0
  176. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/_/__next.!KGFwcCk.traces.$d$traceId.__PAGE__.txt +27 -0
  177. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/_/__next._full.txt +19 -0
  178. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/_/__next._tree.txt +2 -0
  179. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/_/index.html +1 -0
  180. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/_/index.txt +19 -0
  181. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/__next.!KGFwcCk.traces.__PAGE__.txt +29 -0
  182. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/__next._full.txt +22 -0
  183. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/__next._tree.txt +2 -0
  184. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/index.html +1 -0
  185. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/traces/index.txt +22 -0
  186. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/vercel.svg +1 -0
  187. opengrade_ai_server-0.1.0/src/opengrade_server/_ui/window.svg +1 -0
  188. opengrade_ai_server-0.1.0/src/opengrade_server/api/__init__.py +1 -0
  189. opengrade_ai_server-0.1.0/src/opengrade_server/api/deps.py +780 -0
  190. opengrade_ai_server-0.1.0/src/opengrade_server/api/errors.py +134 -0
  191. opengrade_ai_server-0.1.0/src/opengrade_server/api/json_guard.py +138 -0
  192. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/__init__.py +32 -0
  193. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/app_configs.py +182 -0
  194. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/datasets.py +220 -0
  195. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/evaluators.py +52 -0
  196. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/experiments.py +679 -0
  197. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/live_interactions.py +356 -0
  198. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/metrics_packs.py +116 -0
  199. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/metrics_sets.py +266 -0
  200. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/monitoring.py +92 -0
  201. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/otlp.py +108 -0
  202. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/overview.py +135 -0
  203. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/projects.py +59 -0
  204. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/release.py +48 -0
  205. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/trace_scores.py +144 -0
  206. opengrade_ai_server-0.1.0/src/opengrade_server/api/routes/traces.py +180 -0
  207. opengrade_ai_server-0.1.0/src/opengrade_server/api/schemas.py +1002 -0
  208. opengrade_ai_server-0.1.0/src/opengrade_server/application/__init__.py +1 -0
  209. opengrade_ai_server-0.1.0/src/opengrade_server/application/app_configs.py +149 -0
  210. opengrade_ai_server-0.1.0/src/opengrade_server/application/client_results.py +226 -0
  211. opengrade_ai_server-0.1.0/src/opengrade_server/application/compare.py +351 -0
  212. opengrade_ai_server-0.1.0/src/opengrade_server/application/compare_items.py +261 -0
  213. opengrade_ai_server-0.1.0/src/opengrade_server/application/dataset_import.py +120 -0
  214. opengrade_ai_server-0.1.0/src/opengrade_server/application/datasets.py +343 -0
  215. opengrade_ai_server-0.1.0/src/opengrade_server/application/evaluate.py +382 -0
  216. opengrade_ai_server-0.1.0/src/opengrade_server/application/evaluators.py +84 -0
  217. opengrade_ai_server-0.1.0/src/opengrade_server/application/experiment_metrics.py +83 -0
  218. opengrade_ai_server-0.1.0/src/opengrade_server/application/experiment_outputs.py +131 -0
  219. opengrade_ai_server-0.1.0/src/opengrade_server/application/experiments.py +130 -0
  220. opengrade_ai_server-0.1.0/src/opengrade_server/application/live_interactions.py +650 -0
  221. opengrade_ai_server-0.1.0/src/opengrade_server/application/live_judge_calibration.py +121 -0
  222. opengrade_ai_server-0.1.0/src/opengrade_server/application/metrics_packs.py +242 -0
  223. opengrade_ai_server-0.1.0/src/opengrade_server/application/metrics_sets.py +637 -0
  224. opengrade_ai_server-0.1.0/src/opengrade_server/application/monitoring.py +361 -0
  225. opengrade_ai_server-0.1.0/src/opengrade_server/application/project_overview.py +411 -0
  226. opengrade_ai_server-0.1.0/src/opengrade_server/application/projects.py +66 -0
  227. opengrade_ai_server-0.1.0/src/opengrade_server/application/release_check.py +200 -0
  228. opengrade_ai_server-0.1.0/src/opengrade_server/application/scoring_inputs.py +104 -0
  229. opengrade_ai_server-0.1.0/src/opengrade_server/application/scoring_recovery.py +100 -0
  230. opengrade_ai_server-0.1.0/src/opengrade_server/application/trace_scores.py +93 -0
  231. opengrade_ai_server-0.1.0/src/opengrade_server/application/traces.py +190 -0
  232. opengrade_ai_server-0.1.0/src/opengrade_server/cli.py +217 -0
  233. opengrade_ai_server-0.1.0/src/opengrade_server/config.py +39 -0
  234. opengrade_ai_server-0.1.0/src/opengrade_server/domain/__init__.py +1 -0
  235. opengrade_ai_server-0.1.0/src/opengrade_server/domain/app_config.py +101 -0
  236. opengrade_ai_server-0.1.0/src/opengrade_server/domain/dataset.py +82 -0
  237. opengrade_ai_server-0.1.0/src/opengrade_server/domain/evaluation.py +99 -0
  238. opengrade_ai_server-0.1.0/src/opengrade_server/domain/evaluator.py +53 -0
  239. opengrade_ai_server-0.1.0/src/opengrade_server/domain/experiment.py +67 -0
  240. opengrade_ai_server-0.1.0/src/opengrade_server/domain/experiment_metric.py +37 -0
  241. opengrade_ai_server-0.1.0/src/opengrade_server/domain/experiment_output.py +56 -0
  242. opengrade_ai_server-0.1.0/src/opengrade_server/domain/live_interaction.py +191 -0
  243. opengrade_ai_server-0.1.0/src/opengrade_server/domain/live_overview.py +48 -0
  244. opengrade_ai_server-0.1.0/src/opengrade_server/domain/metrics_pack.py +193 -0
  245. opengrade_ai_server-0.1.0/src/opengrade_server/domain/metrics_set.py +263 -0
  246. opengrade_ai_server-0.1.0/src/opengrade_server/domain/project.py +39 -0
  247. opengrade_ai_server-0.1.0/src/opengrade_server/domain/rag_qa.py +55 -0
  248. opengrade_ai_server-0.1.0/src/opengrade_server/domain/repositories.py +281 -0
  249. opengrade_ai_server-0.1.0/src/opengrade_server/domain/retrieval.py +67 -0
  250. opengrade_ai_server-0.1.0/src/opengrade_server/domain/task_types.py +85 -0
  251. opengrade_ai_server-0.1.0/src/opengrade_server/domain/trace.py +38 -0
  252. opengrade_ai_server-0.1.0/src/opengrade_server/domain/trace_score.py +158 -0
  253. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/__init__.py +50 -0
  254. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/deterministic.py +486 -0
  255. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/__init__.py +34 -0
  256. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/cache.py +27 -0
  257. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/claims.py +178 -0
  258. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/errors.py +35 -0
  259. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/evaluators.py +632 -0
  260. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/parsing.py +172 -0
  261. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/prompts.py +650 -0
  262. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/rubric.py +17 -0
  263. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/schemas.py +94 -0
  264. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/snapshots.py +66 -0
  265. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/judges/warnings.py +48 -0
  266. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/outcomes.py +82 -0
  267. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/protocol.py +46 -0
  268. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/registry.py +40 -0
  269. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/runner.py +212 -0
  270. opengrade_ai_server-0.1.0/src/opengrade_server/evaluation/trace_context.py +157 -0
  271. opengrade_ai_server-0.1.0/src/opengrade_server/infrastructure/__init__.py +1 -0
  272. opengrade_ai_server-0.1.0/src/opengrade_server/infrastructure/db.py +123 -0
  273. opengrade_ai_server-0.1.0/src/opengrade_server/infrastructure/llm.py +49 -0
  274. opengrade_ai_server-0.1.0/src/opengrade_server/infrastructure/models.py +679 -0
  275. opengrade_ai_server-0.1.0/src/opengrade_server/infrastructure/repositories.py +1685 -0
  276. opengrade_ai_server-0.1.0/src/opengrade_server/main.py +66 -0
  277. opengrade_ai_server-0.1.0/src/opengrade_server/regression/__init__.py +52 -0
  278. opengrade_ai_server-0.1.0/src/opengrade_server/regression/aggregate.py +276 -0
  279. opengrade_ai_server-0.1.0/src/opengrade_server/regression/bootstrap.py +50 -0
  280. opengrade_ai_server-0.1.0/src/opengrade_server/regression/policy.py +462 -0
  281. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/__init__.py +1 -0
  282. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/ingestion.py +24 -0
  283. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/live_binding.py +217 -0
  284. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/normalization.py +128 -0
  285. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/openinference.py +72 -0
  286. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/otel.py +197 -0
  287. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/redaction.py +57 -0
  288. opengrade_ai_server-0.1.0/src/opengrade_server/tracing/run_binding.py +123 -0
  289. opengrade_ai_server-0.1.0/src/opengrade_server/ui.py +111 -0
  290. opengrade_ai_server-0.1.0/tests/__init__.py +0 -0
  291. opengrade_ai_server-0.1.0/tests/conftest.py +32 -0
  292. opengrade_ai_server-0.1.0/tests/fixtures/evals/scoring_docs.json +13 -0
  293. opengrade_ai_server-0.1.0/tests/fixtures/judge_prompts/system_templates.json +14 -0
  294. opengrade_ai_server-0.1.0/tests/support/__init__.py +0 -0
  295. opengrade_ai_server-0.1.0/tests/support/fake_llm.py +98 -0
  296. opengrade_ai_server-0.1.0/tests/support/packaged_ui.py +12 -0
  297. opengrade_ai_server-0.1.0/tests/support/postgres.py +119 -0
  298. opengrade_ai_server-0.1.0/tests/support/repositories.py +252 -0
  299. opengrade_ai_server-0.1.0/tests/test_api_app_configs.py +230 -0
  300. opengrade_ai_server-0.1.0/tests/test_api_client_results.py +215 -0
  301. opengrade_ai_server-0.1.0/tests/test_api_compare.py +504 -0
  302. opengrade_ai_server-0.1.0/tests/test_api_compare_items.py +472 -0
  303. opengrade_ai_server-0.1.0/tests/test_api_compare_postgres.py +78 -0
  304. opengrade_ai_server-0.1.0/tests/test_api_dataset_import.py +215 -0
  305. opengrade_ai_server-0.1.0/tests/test_api_evaluation.py +778 -0
  306. opengrade_ai_server-0.1.0/tests/test_api_evaluation_postgres.py +73 -0
  307. opengrade_ai_server-0.1.0/tests/test_api_experiment_metrics.py +227 -0
  308. opengrade_ai_server-0.1.0/tests/test_api_experiment_outputs.py +376 -0
  309. opengrade_ai_server-0.1.0/tests/test_api_experiments_app_config.py +262 -0
  310. opengrade_ai_server-0.1.0/tests/test_api_live_interactions.py +405 -0
  311. opengrade_ai_server-0.1.0/tests/test_api_metrics_packs.py +256 -0
  312. opengrade_ai_server-0.1.0/tests/test_api_metrics_sets.py +314 -0
  313. opengrade_ai_server-0.1.0/tests/test_api_non_finite_json.py +149 -0
  314. opengrade_ai_server-0.1.0/tests/test_api_otlp.py +249 -0
  315. opengrade_ai_server-0.1.0/tests/test_api_otlp_live.py +275 -0
  316. opengrade_ai_server-0.1.0/tests/test_api_otlp_run_binding.py +419 -0
  317. opengrade_ai_server-0.1.0/tests/test_api_overview.py +435 -0
  318. opengrade_ai_server-0.1.0/tests/test_api_projects.py +117 -0
  319. opengrade_ai_server-0.1.0/tests/test_api_projects_postgres.py +58 -0
  320. opengrade_ai_server-0.1.0/tests/test_api_rag_qa_items.py +132 -0
  321. opengrade_ai_server-0.1.0/tests/test_api_release_check.py +563 -0
  322. opengrade_ai_server-0.1.0/tests/test_api_task_types.py +149 -0
  323. opengrade_ai_server-0.1.0/tests/test_api_trace_scores.py +319 -0
  324. opengrade_ai_server-0.1.0/tests/test_api_traces_list_postgres.py +135 -0
  325. opengrade_ai_server-0.1.0/tests/test_api_traces_postgres.py +227 -0
  326. opengrade_ai_server-0.1.0/tests/test_application_evaluate_thresholds.py +225 -0
  327. opengrade_ai_server-0.1.0/tests/test_application_experiment_outputs.py +382 -0
  328. opengrade_ai_server-0.1.0/tests/test_application_experiment_params.py +41 -0
  329. opengrade_ai_server-0.1.0/tests/test_application_live_trace_scores.py +124 -0
  330. opengrade_ai_server-0.1.0/tests/test_application_metrics_sets.py +610 -0
  331. opengrade_ai_server-0.1.0/tests/test_average_precision.py +22 -0
  332. opengrade_ai_server-0.1.0/tests/test_cli_check.py +125 -0
  333. opengrade_ai_server-0.1.0/tests/test_compare_bucket_insufficient_n.py +27 -0
  334. opengrade_ai_server-0.1.0/tests/test_compare_evaluator_names.py +193 -0
  335. opengrade_ai_server-0.1.0/tests/test_dataset_auto_version.py +353 -0
  336. opengrade_ai_server-0.1.0/tests/test_dataset_import_parser.py +83 -0
  337. opengrade_ai_server-0.1.0/tests/test_default_pack_smoke.py +51 -0
  338. opengrade_ai_server-0.1.0/tests/test_domain_app_config.py +66 -0
  339. opengrade_ai_server-0.1.0/tests/test_domain_experiment.py +18 -0
  340. opengrade_ai_server-0.1.0/tests/test_domain_experiment_metric.py +59 -0
  341. opengrade_ai_server-0.1.0/tests/test_domain_experiment_output.py +26 -0
  342. opengrade_ai_server-0.1.0/tests/test_domain_metrics_pack.py +112 -0
  343. opengrade_ai_server-0.1.0/tests/test_domain_metrics_set.py +108 -0
  344. opengrade_ai_server-0.1.0/tests/test_domain_project.py +33 -0
  345. opengrade_ai_server-0.1.0/tests/test_domain_rag_qa.py +52 -0
  346. opengrade_ai_server-0.1.0/tests/test_domain_trace_score.py +89 -0
  347. opengrade_ai_server-0.1.0/tests/test_escape_like.py +11 -0
  348. opengrade_ai_server-0.1.0/tests/test_evaluate_robustness.py +297 -0
  349. opengrade_ai_server-0.1.0/tests/test_evaluation_engine.py +701 -0
  350. opengrade_ai_server-0.1.0/tests/test_hit_at_k.py +206 -0
  351. opengrade_ai_server-0.1.0/tests/test_inmemory_metrics_set_repo.py +19 -0
  352. opengrade_ai_server-0.1.0/tests/test_judge_claim_cache_postgres.py +48 -0
  353. opengrade_ai_server-0.1.0/tests/test_judge_sanity_variants.py +133 -0
  354. opengrade_ai_server-0.1.0/tests/test_judges_cache.py +42 -0
  355. opengrade_ai_server-0.1.0/tests/test_judges_claims.py +106 -0
  356. opengrade_ai_server-0.1.0/tests/test_judges_evaluators.py +644 -0
  357. opengrade_ai_server-0.1.0/tests/test_judges_parsing.py +198 -0
  358. opengrade_ai_server-0.1.0/tests/test_judges_prompts.py +146 -0
  359. opengrade_ai_server-0.1.0/tests/test_judges_rubric.py +27 -0
  360. opengrade_ai_server-0.1.0/tests/test_judges_snapshots.py +45 -0
  361. opengrade_ai_server-0.1.0/tests/test_judges_warnings.py +94 -0
  362. opengrade_ai_server-0.1.0/tests/test_live_interactions.py +850 -0
  363. opengrade_ai_server-0.1.0/tests/test_live_judge_calibration.py +78 -0
  364. opengrade_ai_server-0.1.0/tests/test_live_list_postgres.py +71 -0
  365. opengrade_ai_server-0.1.0/tests/test_llm_client.py +96 -0
  366. opengrade_ai_server-0.1.0/tests/test_metrics_sets_postgres.py +181 -0
  367. opengrade_ai_server-0.1.0/tests/test_migration_0008.py +46 -0
  368. opengrade_ai_server-0.1.0/tests/test_migration_0009.py +105 -0
  369. opengrade_ai_server-0.1.0/tests/test_mrr.py +86 -0
  370. opengrade_ai_server-0.1.0/tests/test_must_contain.py +82 -0
  371. opengrade_ai_server-0.1.0/tests/test_otlp_split_batch_postgres.py +102 -0
  372. opengrade_ai_server-0.1.0/tests/test_outcomes.py +62 -0
  373. opengrade_ai_server-0.1.0/tests/test_overview_buckets.py +29 -0
  374. opengrade_ai_server-0.1.0/tests/test_policy.py +227 -0
  375. opengrade_ai_server-0.1.0/tests/test_postgres_experiment_metric_repository.py +54 -0
  376. opengrade_ai_server-0.1.0/tests/test_postgres_trace_score_repository.py +105 -0
  377. opengrade_ai_server-0.1.0/tests/test_production_monitoring.py +177 -0
  378. opengrade_ai_server-0.1.0/tests/test_project_overview_use_case.py +457 -0
  379. opengrade_ai_server-0.1.0/tests/test_recall_at_k.py +87 -0
  380. opengrade_ai_server-0.1.0/tests/test_regression.py +433 -0
  381. opengrade_ai_server-0.1.0/tests/test_release_insufficient_n.py +185 -0
  382. opengrade_ai_server-0.1.0/tests/test_release_policy_experiment_metrics.py +61 -0
  383. opengrade_ai_server-0.1.0/tests/test_rename_contract.py +21 -0
  384. opengrade_ai_server-0.1.0/tests/test_run_binding.py +134 -0
  385. opengrade_ai_server-0.1.0/tests/test_sqlite_experiment_metric_repository.py +35 -0
  386. opengrade_ai_server-0.1.0/tests/test_sqlite_smoke.py +179 -0
  387. opengrade_ai_server-0.1.0/tests/test_sqlite_trace_score_repository.py +119 -0
  388. opengrade_ai_server-0.1.0/tests/test_stale_runs_postgres.py +152 -0
  389. opengrade_ai_server-0.1.0/tests/test_tracing_normalization.py +157 -0
  390. opengrade_ai_server-0.1.0/tests/test_ui_resolve.py +88 -0
  391. opengrade_ai_server-0.1.0/uv.lock +3016 -0
@@ -0,0 +1,8 @@
1
+ .venv
2
+ __pycache__
3
+ *.pyc
4
+ .pytest_cache
5
+ .mypy_cache
6
+ .ruff_cache
7
+ .git
8
+ .env
@@ -0,0 +1,36 @@
1
+ # Copy to backend/.env and adjust as needed:
2
+ # cp .env.example .env
3
+
4
+ # Database (Compose publishes Postgres on host port 5434)
5
+ DATABASE_URL=postgresql+asyncpg://opengrade:opengrade@localhost:5434/opengrade
6
+
7
+ APP_HOST=0.0.0.0
8
+ APP_PORT=8000
9
+
10
+ # Capture prompt/completion content on traces (off by default).
11
+ # Required for portfolio demo from-trace (OTLP input.value / output.value).
12
+ CONTENT_CAPTURE_ENABLED=false
13
+
14
+ # OTLP ingestion body size limit (bytes)
15
+ OTLP_MAX_BODY_BYTES=4194304
16
+
17
+ # App-wide JSON request body buffer cap (bytes). Middleware rejects larger JSON
18
+ # before parsing. Raise this if you raise OTLP_MAX_BODY_BYTES for JSON OTLP.
19
+ JSON_MAX_BODY_BYTES=4194304
20
+
21
+ # --- LLM judges (LiteLLM) ---
22
+ # Cloud example (Google AI Studio; default when LLM_MODEL is unset):
23
+ LLM_MODEL=gemini/gemini-3.8-flash
24
+ LLM_API_KEY=
25
+
26
+ # Local Ollama example:
27
+ # LLM_MODEL=ollama/gemma4:e2b
28
+ # From Docker Compose API use host.docker.internal (extra_hosts in compose).
29
+ # From a host-local API process use http://localhost:11434 instead.
30
+ # LLM_API_BASE=http://host.docker.internal:11434
31
+ # LLM_API_KEY=
32
+
33
+ # Local Ollama cold-start can exceed 60s on small hosts
34
+ LLM_TIMEOUT_SECONDS=300
35
+ LLM_MAX_RETRIES=2
36
+ LLM_MAX_CONCURRENCY=8
@@ -0,0 +1,42 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ .venv/
8
+ venv/
9
+ .env
10
+ .env.local
11
+ *.egg-info/
12
+ dist/
13
+ build/
14
+ .mypy_cache/
15
+ .ruff_cache/
16
+ .pytest_cache/
17
+ .coverage
18
+ htmlcov/
19
+ .uv/
20
+
21
+ # IDE / OS
22
+ .idea/
23
+ .vscode/
24
+ *.swp
25
+ .DS_Store
26
+ Thumbs.db
27
+
28
+ # Docker volumes
29
+ postgres_data/
30
+
31
+ .worktrees/
32
+ .cursor/
33
+ docs/superpowers/
34
+
35
+ # Packaged Next.js UI (built by scripts/package-ui.sh)
36
+ backend/src/opengrade_server/_ui/**
37
+ !backend/src/opengrade_server/_ui/.gitkeep
38
+
39
+ opengrade.db
40
+ **/opengrade.db
41
+
42
+ .opengrade/
@@ -0,0 +1,36 @@
1
+ # One image for API and UI: the static UI export is built here and served by
2
+ # FastAPI, exactly like `opengrade ui` serves the packaged wheel.
3
+
4
+ FROM node:22-alpine AS ui
5
+ WORKDIR /frontend
6
+ COPY frontend/package.json frontend/package-lock.json ./
7
+ RUN npm ci
8
+ COPY frontend/ ./
9
+ ENV NEXT_TELEMETRY_DISABLED=1
10
+ RUN npm run build
11
+
12
+ FROM python:3.12-slim
13
+
14
+ WORKDIR /app
15
+
16
+ RUN pip install --no-cache-dir uv
17
+
18
+ COPY sdk/pyproject.toml sdk/README.md sdk/hatch_build.py /sdk/
19
+ COPY sdk/src /sdk/src
20
+
21
+ COPY backend/pyproject.toml backend/README.md backend/hatch_build.py ./
22
+ COPY backend/src ./src
23
+ COPY --from=ui /frontend/out ./src/opengrade_server/_ui
24
+ COPY backend/alembic ./alembic
25
+ COPY backend/alembic.ini ./
26
+ COPY backend/docker-entrypoint.sh ./
27
+
28
+ RUN uv pip install --system --no-cache . \
29
+ && python -c "from pathlib import Path; p = Path('docker-entrypoint.sh'); data = p.read_bytes().replace(b'\r\n', b'\n').replace(b'\r', b'\n').lstrip(b'\xef\xbb\xbf'); p.write_bytes(data); p.chmod(0o755)"
30
+
31
+ ENV APP_HOST=0.0.0.0
32
+ ENV APP_PORT=8000
33
+
34
+ EXPOSE 8000
35
+
36
+ ENTRYPOINT ["/app/docker-entrypoint.sh"]
@@ -0,0 +1,55 @@
1
+ Metadata-Version: 2.5
2
+ Name: opengrade-ai-server
3
+ Version: 0.1.0
4
+ Summary: AI Evaluation & Observability Platform server (FastAPI backend)
5
+ Requires-Python: >=3.12
6
+ Requires-Dist: aiosqlite>=0.20.0
7
+ Requires-Dist: alembic>=1.14.0
8
+ Requires-Dist: asyncpg>=0.30.0
9
+ Requires-Dist: fastapi>=0.115.0
10
+ Requires-Dist: httpx2>=2.13.0
11
+ Requires-Dist: jsonschema>=4.23.0
12
+ Requires-Dist: litellm>=1.55.0
13
+ Requires-Dist: opengrade-ai==0.1.0
14
+ Requires-Dist: opentelemetry-proto>=1.28.0
15
+ Requires-Dist: protobuf>=5.0.0
16
+ Requires-Dist: pydantic-settings>=2.6.0
17
+ Requires-Dist: pydantic>=2.10.0
18
+ Requires-Dist: python-multipart>=0.0.9
19
+ Requires-Dist: pyyaml>=6.0.2
20
+ Requires-Dist: rich>=13.9.0
21
+ Requires-Dist: sqlalchemy[asyncio]>=2.0.36
22
+ Requires-Dist: typer>=0.15.0
23
+ Requires-Dist: uvicorn[standard]>=0.32.0
24
+ Provides-Extra: dev
25
+ Requires-Dist: mypy>=1.13.0; extra == 'dev'
26
+ Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
27
+ Requires-Dist: pytest>=8.3.0; extra == 'dev'
28
+ Requires-Dist: ruff>=0.8.0; extra == 'dev'
29
+ Description-Content-Type: text/markdown
30
+
31
+ # opengrade-ai-server
32
+
33
+ Server package for the AI Evaluation & Observability Platform.
34
+
35
+ - Distribution: `opengrade-ai-server` (PyPI)
36
+ - Import path: `opengrade_server` (e.g. `uvicorn opengrade_server.main:app`)
37
+ - Console scripts: `opengrade` (`ui`, `check`; provided by the `opengrade-ai` dependency) and `opengrade-server` (same commands)
38
+
39
+ ## Local UI
40
+
41
+ ```bash
42
+ # from repo root — build the Next static export into the package
43
+ ./scripts/package-ui.sh
44
+ cd backend && uv sync
45
+ uv run opengrade ui
46
+ # → http://127.0.0.1:8000 (SQLite ./opengrade.db by default)
47
+ ```
48
+
49
+ Postgres:
50
+
51
+ ```bash
52
+ uv run opengrade ui --backend-store-uri 'postgresql+asyncpg://opengrade:opengrade@localhost:5434/opengrade'
53
+ ```
54
+
55
+ See the repository root README for Docker Compose and SDK quickstart.
@@ -0,0 +1,25 @@
1
+ # opengrade-ai-server
2
+
3
+ Server package for the AI Evaluation & Observability Platform.
4
+
5
+ - Distribution: `opengrade-ai-server` (PyPI)
6
+ - Import path: `opengrade_server` (e.g. `uvicorn opengrade_server.main:app`)
7
+ - Console scripts: `opengrade` (`ui`, `check`; provided by the `opengrade-ai` dependency) and `opengrade-server` (same commands)
8
+
9
+ ## Local UI
10
+
11
+ ```bash
12
+ # from repo root — build the Next static export into the package
13
+ ./scripts/package-ui.sh
14
+ cd backend && uv sync
15
+ uv run opengrade ui
16
+ # → http://127.0.0.1:8000 (SQLite ./opengrade.db by default)
17
+ ```
18
+
19
+ Postgres:
20
+
21
+ ```bash
22
+ uv run opengrade ui --backend-store-uri 'postgresql+asyncpg://opengrade:opengrade@localhost:5434/opengrade'
23
+ ```
24
+
25
+ See the repository root README for Docker Compose and SDK quickstart.
@@ -0,0 +1,65 @@
1
+ from logging.config import fileConfig
2
+
3
+ from alembic import context
4
+ from sqlalchemy import pool
5
+ from sqlalchemy.engine import Connection
6
+ from sqlalchemy.ext.asyncio import async_engine_from_config
7
+
8
+ from opengrade_server.config import get_settings
9
+ from opengrade_server.infrastructure.models import Base
10
+
11
+ config = context.config
12
+ if config.config_file_name is not None:
13
+ fileConfig(config.config_file_name)
14
+
15
+ target_metadata = Base.metadata
16
+
17
+
18
+ def get_url() -> str:
19
+ return get_settings().database_url
20
+
21
+
22
+ def run_migrations_offline() -> None:
23
+ url = get_url()
24
+ context.configure(
25
+ url=url,
26
+ target_metadata=target_metadata,
27
+ literal_binds=True,
28
+ dialect_opts={"paramstyle": "named"},
29
+ )
30
+
31
+ with context.begin_transaction():
32
+ context.run_migrations()
33
+
34
+
35
+ def do_run_migrations(connection: Connection) -> None:
36
+ context.configure(connection=connection, target_metadata=target_metadata)
37
+ with context.begin_transaction():
38
+ context.run_migrations()
39
+
40
+
41
+ async def run_async_migrations() -> None:
42
+ configuration = config.get_section(config.config_ini_section) or {}
43
+ configuration["sqlalchemy.url"] = get_url()
44
+ connectable = async_engine_from_config(
45
+ configuration,
46
+ prefix="sqlalchemy.",
47
+ poolclass=pool.NullPool,
48
+ )
49
+
50
+ async with connectable.connect() as connection:
51
+ await connection.run_sync(do_run_migrations)
52
+
53
+ await connectable.dispose()
54
+
55
+
56
+ def run_migrations_online() -> None:
57
+ import asyncio
58
+
59
+ asyncio.run(run_async_migrations())
60
+
61
+
62
+ if context.is_offline_mode():
63
+ run_migrations_offline()
64
+ else:
65
+ run_migrations_online()
@@ -0,0 +1,18 @@
1
+ """Alembic script template."""
2
+
3
+ from alembic import op
4
+ import sqlalchemy as sa
5
+
6
+ # revision identifiers, used by Alembic.
7
+ revision = None
8
+ down_revision = None
9
+ branch_labels = None
10
+ depends_on = None
11
+
12
+
13
+ def upgrade() -> None:
14
+ pass
15
+
16
+
17
+ def downgrade() -> None:
18
+ pass
@@ -0,0 +1,39 @@
1
+ """create projects table
2
+
3
+ Revision ID: 0001_create_projects
4
+ Revises:
5
+ Create Date: 2026-09-30
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Sequence
11
+
12
+ import sqlalchemy as sa
13
+ from alembic import op
14
+
15
+ revision: str = "0001_create_projects"
16
+ down_revision: str | None = None
17
+ branch_labels: str | Sequence[str] | None = None
18
+ depends_on: str | Sequence[str] | None = None
19
+
20
+
21
+ def upgrade() -> None:
22
+ op.create_table(
23
+ "projects",
24
+ sa.Column("id", sa.Uuid(), nullable=False),
25
+ sa.Column("name", sa.String(length=200), nullable=False),
26
+ sa.Column("slug", sa.String(length=200), nullable=False),
27
+ sa.Column(
28
+ "created_at",
29
+ sa.DateTime(timezone=True),
30
+ server_default=sa.text("now()"),
31
+ nullable=False,
32
+ ),
33
+ sa.PrimaryKeyConstraint("id"),
34
+ sa.UniqueConstraint("slug"),
35
+ )
36
+
37
+
38
+ def downgrade() -> None:
39
+ op.drop_table("projects")
@@ -0,0 +1,102 @@
1
+ """create traces and spans tables
2
+
3
+ Revision ID: 0002_create_traces_spans
4
+ Revises: 0001_create_projects
5
+ Create Date: 2026-09-30
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Sequence
11
+
12
+ import sqlalchemy as sa
13
+ from alembic import op
14
+ from sqlalchemy.dialects import postgresql
15
+
16
+ revision: str = "0002_create_traces_spans"
17
+ down_revision: str | None = "0001_create_projects"
18
+ branch_labels: str | Sequence[str] | None = None
19
+ depends_on: str | Sequence[str] | None = None
20
+
21
+
22
+ def upgrade() -> None:
23
+ op.create_table(
24
+ "traces",
25
+ sa.Column("id", sa.Uuid(), nullable=False),
26
+ sa.Column("project_id", sa.Uuid(), nullable=False),
27
+ sa.Column("trace_id", sa.String(length=32), nullable=False),
28
+ sa.Column("name", sa.String(length=512), nullable=False),
29
+ sa.Column("status", sa.String(length=32), nullable=False),
30
+ sa.Column("start_time", sa.DateTime(timezone=True), nullable=False),
31
+ sa.Column("end_time", sa.DateTime(timezone=True), nullable=True),
32
+ sa.Column("input", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
33
+ sa.Column("output", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
34
+ sa.Column(
35
+ "metadata",
36
+ postgresql.JSONB(astext_type=sa.Text()),
37
+ server_default=sa.text("'{}'::jsonb"),
38
+ nullable=False,
39
+ ),
40
+ sa.Column("environment", sa.String(length=200), nullable=True),
41
+ sa.Column("user_id", sa.String(length=200), nullable=True),
42
+ sa.Column("session_id", sa.String(length=200), nullable=True),
43
+ sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"),
44
+ sa.PrimaryKeyConstraint("id"),
45
+ sa.UniqueConstraint("project_id", "trace_id", name="uq_traces_project_trace_id"),
46
+ )
47
+ op.create_index(
48
+ "ix_traces_project_start_time",
49
+ "traces",
50
+ ["project_id", "start_time"],
51
+ unique=False,
52
+ )
53
+ op.create_index(
54
+ "ix_traces_project_status",
55
+ "traces",
56
+ ["project_id", "status"],
57
+ unique=False,
58
+ )
59
+
60
+ op.create_table(
61
+ "spans",
62
+ sa.Column("id", sa.Uuid(), nullable=False),
63
+ sa.Column("trace_pk", sa.Uuid(), nullable=False),
64
+ sa.Column("project_id", sa.Uuid(), nullable=False),
65
+ sa.Column("span_id", sa.String(length=16), nullable=False),
66
+ sa.Column("parent_span_id", sa.String(length=16), nullable=True),
67
+ sa.Column("name", sa.String(length=512), nullable=False),
68
+ sa.Column("kind", sa.String(length=64), nullable=False),
69
+ sa.Column("start_time", sa.DateTime(timezone=True), nullable=False),
70
+ sa.Column("end_time", sa.DateTime(timezone=True), nullable=True),
71
+ sa.Column("status", sa.String(length=32), nullable=False),
72
+ sa.Column(
73
+ "attributes",
74
+ postgresql.JSONB(astext_type=sa.Text()),
75
+ server_default=sa.text("'{}'::jsonb"),
76
+ nullable=False,
77
+ ),
78
+ sa.Column(
79
+ "events",
80
+ postgresql.JSONB(astext_type=sa.Text()),
81
+ server_default=sa.text("'[]'::jsonb"),
82
+ nullable=False,
83
+ ),
84
+ sa.ForeignKeyConstraint(["trace_pk"], ["traces.id"], ondelete="CASCADE"),
85
+ sa.PrimaryKeyConstraint("id"),
86
+ )
87
+ op.create_index("ix_spans_trace_pk", "spans", ["trace_pk"], unique=False)
88
+ op.create_index(
89
+ "ix_spans_project_start_time",
90
+ "spans",
91
+ ["project_id", "start_time"],
92
+ unique=False,
93
+ )
94
+
95
+
96
+ def downgrade() -> None:
97
+ op.drop_index("ix_spans_project_start_time", table_name="spans")
98
+ op.drop_index("ix_spans_trace_pk", table_name="spans")
99
+ op.drop_table("spans")
100
+ op.drop_index("ix_traces_project_status", table_name="traces")
101
+ op.drop_index("ix_traces_project_start_time", table_name="traces")
102
+ op.drop_table("traces")
@@ -0,0 +1,190 @@
1
+ """create evaluation engine tables
2
+
3
+ Revision ID: 0003_create_evaluation_tables
4
+ Revises: 0002_create_traces_spans
5
+ Create Date: 2026-10-01
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Sequence
11
+
12
+ import sqlalchemy as sa
13
+ from alembic import op
14
+ from sqlalchemy.dialects import postgresql
15
+
16
+ revision: str = "0003_create_evaluation_tables"
17
+ down_revision: str | None = "0002_create_traces_spans"
18
+ branch_labels: str | Sequence[str] | None = None
19
+ depends_on: str | Sequence[str] | None = None
20
+
21
+
22
+ def upgrade() -> None:
23
+ op.create_table(
24
+ "datasets",
25
+ sa.Column("id", sa.Uuid(), nullable=False),
26
+ sa.Column("project_id", sa.Uuid(), nullable=False),
27
+ sa.Column("name", sa.String(length=200), nullable=False),
28
+ sa.Column("version", sa.Integer(), server_default="1", nullable=False),
29
+ sa.Column("description", sa.String(length=2000), nullable=True),
30
+ sa.Column(
31
+ "created_at",
32
+ sa.DateTime(timezone=True),
33
+ server_default=sa.text("now()"),
34
+ nullable=False,
35
+ ),
36
+ sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"),
37
+ sa.PrimaryKeyConstraint("id"),
38
+ sa.UniqueConstraint(
39
+ "project_id", "name", "version", name="uq_datasets_project_name_version"
40
+ ),
41
+ )
42
+ op.create_index(
43
+ "ix_datasets_project_created_at",
44
+ "datasets",
45
+ ["project_id", "created_at"],
46
+ unique=False,
47
+ )
48
+
49
+ op.create_table(
50
+ "dataset_items",
51
+ sa.Column("id", sa.Uuid(), nullable=False),
52
+ sa.Column("dataset_id", sa.Uuid(), nullable=False),
53
+ sa.Column("input", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
54
+ sa.Column("expected_output", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
55
+ sa.Column("actual_output", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
56
+ sa.Column("context", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
57
+ sa.Column(
58
+ "metadata",
59
+ postgresql.JSONB(astext_type=sa.Text()),
60
+ server_default=sa.text("'{}'::jsonb"),
61
+ nullable=False,
62
+ ),
63
+ sa.Column("source_trace_id", sa.String(length=32), nullable=True),
64
+ sa.Column("source_span_id", sa.String(length=16), nullable=True),
65
+ sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"], ondelete="CASCADE"),
66
+ sa.PrimaryKeyConstraint("id"),
67
+ )
68
+ op.create_index("ix_dataset_items_dataset_id", "dataset_items", ["dataset_id"], unique=False)
69
+
70
+ op.create_table(
71
+ "evaluators",
72
+ sa.Column("id", sa.Uuid(), nullable=False),
73
+ sa.Column("project_id", sa.Uuid(), nullable=False),
74
+ sa.Column("name", sa.String(length=200), nullable=False),
75
+ sa.Column("type", sa.String(length=32), nullable=False),
76
+ sa.Column(
77
+ "config",
78
+ postgresql.JSONB(astext_type=sa.Text()),
79
+ server_default=sa.text("'{}'::jsonb"),
80
+ nullable=False,
81
+ ),
82
+ sa.Column("version", sa.Integer(), server_default="1", nullable=False),
83
+ sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"),
84
+ sa.PrimaryKeyConstraint("id"),
85
+ sa.UniqueConstraint(
86
+ "project_id", "name", "version", name="uq_evaluators_project_name_version"
87
+ ),
88
+ )
89
+ op.create_index("ix_evaluators_project_id", "evaluators", ["project_id"], unique=False)
90
+
91
+ op.create_table(
92
+ "experiments",
93
+ sa.Column("id", sa.Uuid(), nullable=False),
94
+ sa.Column("project_id", sa.Uuid(), nullable=False),
95
+ sa.Column("name", sa.String(length=200), nullable=False),
96
+ sa.Column("dataset_id", sa.Uuid(), nullable=False),
97
+ sa.Column(
98
+ "model_config",
99
+ postgresql.JSONB(astext_type=sa.Text()),
100
+ server_default=sa.text("'{}'::jsonb"),
101
+ nullable=False,
102
+ ),
103
+ sa.Column("version", sa.String(length=200), nullable=True),
104
+ sa.Column("baseline_experiment_id", sa.Uuid(), nullable=True),
105
+ sa.Column("status", sa.String(length=32), server_default="created", nullable=False),
106
+ sa.Column(
107
+ "created_at",
108
+ sa.DateTime(timezone=True),
109
+ server_default=sa.text("now()"),
110
+ nullable=False,
111
+ ),
112
+ sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"),
113
+ sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"], ondelete="RESTRICT"),
114
+ sa.ForeignKeyConstraint(
115
+ ["baseline_experiment_id"], ["experiments.id"], ondelete="SET NULL"
116
+ ),
117
+ sa.PrimaryKeyConstraint("id"),
118
+ )
119
+ op.create_index(
120
+ "ix_experiments_project_created_at",
121
+ "experiments",
122
+ ["project_id", "created_at"],
123
+ unique=False,
124
+ )
125
+
126
+ op.create_table(
127
+ "evaluation_runs",
128
+ sa.Column("id", sa.Uuid(), nullable=False),
129
+ sa.Column("experiment_id", sa.Uuid(), nullable=False),
130
+ sa.Column("evaluator_id", sa.Uuid(), nullable=False),
131
+ sa.Column("status", sa.String(length=32), nullable=False),
132
+ sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
133
+ sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
134
+ sa.Column(
135
+ "metadata",
136
+ postgresql.JSONB(astext_type=sa.Text()),
137
+ server_default=sa.text("'{}'::jsonb"),
138
+ nullable=False,
139
+ ),
140
+ sa.ForeignKeyConstraint(["experiment_id"], ["experiments.id"], ondelete="CASCADE"),
141
+ sa.ForeignKeyConstraint(["evaluator_id"], ["evaluators.id"], ondelete="RESTRICT"),
142
+ sa.PrimaryKeyConstraint("id"),
143
+ )
144
+ op.create_index(
145
+ "ix_evaluation_runs_experiment_id",
146
+ "evaluation_runs",
147
+ ["experiment_id"],
148
+ unique=False,
149
+ )
150
+
151
+ op.create_table(
152
+ "evaluation_results",
153
+ sa.Column("id", sa.Uuid(), nullable=False),
154
+ sa.Column("run_id", sa.Uuid(), nullable=False),
155
+ sa.Column("dataset_item_id", sa.Uuid(), nullable=False),
156
+ sa.Column("score", sa.Float(), nullable=True),
157
+ sa.Column("label", sa.String(length=64), nullable=True),
158
+ sa.Column("explanation", sa.String(length=4000), nullable=True),
159
+ sa.Column(
160
+ "metadata",
161
+ postgresql.JSONB(astext_type=sa.Text()),
162
+ server_default=sa.text("'{}'::jsonb"),
163
+ nullable=False,
164
+ ),
165
+ sa.Column("duration_ms", sa.Integer(), nullable=True),
166
+ sa.ForeignKeyConstraint(["run_id"], ["evaluation_runs.id"], ondelete="CASCADE"),
167
+ sa.ForeignKeyConstraint(["dataset_item_id"], ["dataset_items.id"], ondelete="CASCADE"),
168
+ sa.PrimaryKeyConstraint("id"),
169
+ )
170
+ op.create_index(
171
+ "ix_evaluation_results_run_item",
172
+ "evaluation_results",
173
+ ["run_id", "dataset_item_id"],
174
+ unique=False,
175
+ )
176
+
177
+
178
+ def downgrade() -> None:
179
+ op.drop_index("ix_evaluation_results_run_item", table_name="evaluation_results")
180
+ op.drop_table("evaluation_results")
181
+ op.drop_index("ix_evaluation_runs_experiment_id", table_name="evaluation_runs")
182
+ op.drop_table("evaluation_runs")
183
+ op.drop_index("ix_experiments_project_created_at", table_name="experiments")
184
+ op.drop_table("experiments")
185
+ op.drop_index("ix_evaluators_project_id", table_name="evaluators")
186
+ op.drop_table("evaluators")
187
+ op.drop_index("ix_dataset_items_dataset_id", table_name="dataset_items")
188
+ op.drop_table("dataset_items")
189
+ op.drop_index("ix_datasets_project_created_at", table_name="datasets")
190
+ op.drop_table("datasets")
@@ -0,0 +1,60 @@
1
+ """rename experiments.application_version to version
2
+
3
+ Revision ID: 0004_rename_experiment_version
4
+ Revises: 0003_create_evaluation_tables
5
+ Create Date: 2026-10-02
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Sequence
11
+
12
+ from alembic import op
13
+
14
+ revision: str = "0004_rename_experiment_version"
15
+ down_revision: str | None = "0003_create_evaluation_tables"
16
+ branch_labels: str | Sequence[str] | None = None
17
+ depends_on: str | Sequence[str] | None = None
18
+
19
+
20
+ def upgrade() -> None:
21
+ op.execute(
22
+ """
23
+ DO $$
24
+ BEGIN
25
+ IF EXISTS (
26
+ SELECT 1
27
+ FROM information_schema.columns
28
+ WHERE table_name = 'experiments'
29
+ AND column_name = 'application_version'
30
+ ) THEN
31
+ ALTER TABLE experiments
32
+ RENAME COLUMN application_version TO version;
33
+ END IF;
34
+ END $$;
35
+ """
36
+ )
37
+
38
+
39
+ def downgrade() -> None:
40
+ op.execute(
41
+ """
42
+ DO $$
43
+ BEGIN
44
+ IF EXISTS (
45
+ SELECT 1
46
+ FROM information_schema.columns
47
+ WHERE table_name = 'experiments'
48
+ AND column_name = 'version'
49
+ ) AND NOT EXISTS (
50
+ SELECT 1
51
+ FROM information_schema.columns
52
+ WHERE table_name = 'experiments'
53
+ AND column_name = 'application_version'
54
+ ) THEN
55
+ ALTER TABLE experiments
56
+ RENAME COLUMN version TO application_version;
57
+ END IF;
58
+ END $$;
59
+ """
60
+ )