provekit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (485) hide show
  1. provekit/__init__.py +0 -0
  2. provekit/cli.py +256 -0
  3. provekit/config.py +77 -0
  4. provekit/database.py +132 -0
  5. provekit/main.py +115 -0
  6. provekit/migrations/env.py +40 -0
  7. provekit/migrations/script.py.mako +22 -0
  8. provekit/migrations/versions/323eb73d463c_user_email_verified.py +32 -0
  9. provekit/migrations/versions/49e8ab812556_baseline_schema.py +241 -0
  10. provekit/migrations/versions/819ed5ff183e_workspace_ingest_key.py +35 -0
  11. provekit/migrations/versions/a1b2c3d4e5f6_user_token_version.py +27 -0
  12. provekit/models.py +191 -0
  13. provekit/observability.py +154 -0
  14. provekit/routers/__init__.py +0 -0
  15. provekit/routers/auth.py +150 -0
  16. provekit/routers/connections.py +270 -0
  17. provekit/routers/deployments.py +161 -0
  18. provekit/routers/flows.py +188 -0
  19. provekit/routers/library.py +265 -0
  20. provekit/routers/prompts.py +72 -0
  21. provekit/routers/run.py +203 -0
  22. provekit/routers/runtime.py +103 -0
  23. provekit/routers/traces.py +65 -0
  24. provekit/routers/usage.py +42 -0
  25. provekit/services/__init__.py +0 -0
  26. provekit/services/assertions.py +135 -0
  27. provekit/services/auth.py +131 -0
  28. provekit/services/deploy.py +63 -0
  29. provekit/services/dispatch.py +313 -0
  30. provekit/services/email.py +38 -0
  31. provekit/services/flow.py +299 -0
  32. provekit/services/limits.py +109 -0
  33. provekit/services/masking.py +45 -0
  34. provekit/services/netguard.py +69 -0
  35. provekit/services/otel.py +146 -0
  36. provekit/services/promptfoo.py +92 -0
  37. provekit/services/providers/__init__.py +0 -0
  38. provekit/services/providers/a2a_client.py +110 -0
  39. provekit/services/providers/agent_http.py +55 -0
  40. provekit/services/providers/llm.py +364 -0
  41. provekit/services/providers/mcp_client.py +289 -0
  42. provekit/services/runstore.py +93 -0
  43. provekit/services/sealing.py +106 -0
  44. provekit/services/templates.py +69 -0
  45. provekit/services/testfile.py +122 -0
  46. provekit/services/tooling.py +194 -0
  47. provekit/services/workspace.py +91 -0
  48. provekit/templates/flows/_manifest.json +2594 -0
  49. provekit/templates/flows/banking-action-items.yaml +43 -0
  50. provekit/templates/flows/banking-answer.yaml +43 -0
  51. provekit/templates/flows/banking-classify-intent.yaml +43 -0
  52. provekit/templates/flows/banking-draft-critique-revise.yaml +75 -0
  53. provekit/templates/flows/banking-extract-to-json.yaml +43 -0
  54. provekit/templates/flows/banking-faq-match-answer.yaml +60 -0
  55. provekit/templates/flows/banking-keyword-extract.yaml +43 -0
  56. provekit/templates/flows/banking-language-route.yaml +103 -0
  57. provekit/templates/flows/banking-moderation-gate.yaml +73 -0
  58. provekit/templates/flows/banking-redact-summarize.yaml +59 -0
  59. provekit/templates/flows/banking-score-gate.yaml +73 -0
  60. provekit/templates/flows/banking-sentiment-route.yaml +73 -0
  61. provekit/templates/flows/banking-severity-tiers.yaml +103 -0
  62. provekit/templates/flows/banking-summarize-then-decide.yaml +88 -0
  63. provekit/templates/flows/banking-tag-enrich.yaml +43 -0
  64. provekit/templates/flows/banking-tone-rewrite.yaml +43 -0
  65. provekit/templates/flows/banking-translate-reply.yaml +58 -0
  66. provekit/templates/flows/banking-triage-route.yaml +73 -0
  67. provekit/templates/flows/community-forum-action-items.yaml +43 -0
  68. provekit/templates/flows/community-forum-answer.yaml +43 -0
  69. provekit/templates/flows/community-forum-classify-intent.yaml +43 -0
  70. provekit/templates/flows/community-forum-draft-critique-revise.yaml +75 -0
  71. provekit/templates/flows/community-forum-extract-to-json.yaml +43 -0
  72. provekit/templates/flows/community-forum-faq-match-answer.yaml +60 -0
  73. provekit/templates/flows/community-forum-keyword-extract.yaml +43 -0
  74. provekit/templates/flows/community-forum-language-route.yaml +103 -0
  75. provekit/templates/flows/community-forum-moderation-gate.yaml +73 -0
  76. provekit/templates/flows/community-forum-redact-summarize.yaml +59 -0
  77. provekit/templates/flows/community-forum-score-gate.yaml +73 -0
  78. provekit/templates/flows/community-forum-sentiment-route.yaml +73 -0
  79. provekit/templates/flows/community-forum-severity-tiers.yaml +103 -0
  80. provekit/templates/flows/community-forum-summarize-then-decide.yaml +88 -0
  81. provekit/templates/flows/community-forum-tag-enrich.yaml +43 -0
  82. provekit/templates/flows/community-forum-tone-rewrite.yaml +43 -0
  83. provekit/templates/flows/community-forum-translate-reply.yaml +58 -0
  84. provekit/templates/flows/community-forum-triage-route.yaml +73 -0
  85. provekit/templates/flows/content-moderation-action-items.yaml +43 -0
  86. provekit/templates/flows/content-moderation-answer.yaml +43 -0
  87. provekit/templates/flows/content-moderation-classify-intent.yaml +43 -0
  88. provekit/templates/flows/content-moderation-draft-critique-revise.yaml +75 -0
  89. provekit/templates/flows/content-moderation-extract-to-json.yaml +43 -0
  90. provekit/templates/flows/content-moderation-faq-match-answer.yaml +60 -0
  91. provekit/templates/flows/content-moderation-keyword-extract.yaml +43 -0
  92. provekit/templates/flows/content-moderation-language-route.yaml +103 -0
  93. provekit/templates/flows/content-moderation-moderation-gate.yaml +73 -0
  94. provekit/templates/flows/content-moderation-redact-summarize.yaml +59 -0
  95. provekit/templates/flows/content-moderation-score-gate.yaml +73 -0
  96. provekit/templates/flows/content-moderation-sentiment-route.yaml +73 -0
  97. provekit/templates/flows/content-moderation-severity-tiers.yaml +103 -0
  98. provekit/templates/flows/content-moderation-summarize-then-decide.yaml +88 -0
  99. provekit/templates/flows/content-moderation-tag-enrich.yaml +43 -0
  100. provekit/templates/flows/content-moderation-tone-rewrite.yaml +43 -0
  101. provekit/templates/flows/content-moderation-translate-reply.yaml +58 -0
  102. provekit/templates/flows/content-moderation-triage-route.yaml +73 -0
  103. provekit/templates/flows/customer-support-action-items.yaml +43 -0
  104. provekit/templates/flows/customer-support-answer.yaml +43 -0
  105. provekit/templates/flows/customer-support-classify-intent.yaml +43 -0
  106. provekit/templates/flows/customer-support-draft-critique-revise.yaml +75 -0
  107. provekit/templates/flows/customer-support-extract-to-json.yaml +43 -0
  108. provekit/templates/flows/customer-support-faq-match-answer.yaml +60 -0
  109. provekit/templates/flows/customer-support-keyword-extract.yaml +43 -0
  110. provekit/templates/flows/customer-support-language-route.yaml +103 -0
  111. provekit/templates/flows/customer-support-moderation-gate.yaml +73 -0
  112. provekit/templates/flows/customer-support-redact-summarize.yaml +59 -0
  113. provekit/templates/flows/customer-support-score-gate.yaml +73 -0
  114. provekit/templates/flows/customer-support-sentiment-route.yaml +73 -0
  115. provekit/templates/flows/customer-support-severity-tiers.yaml +103 -0
  116. provekit/templates/flows/customer-support-summarize-then-decide.yaml +88 -0
  117. provekit/templates/flows/customer-support-tag-enrich.yaml +43 -0
  118. provekit/templates/flows/customer-support-tone-rewrite.yaml +43 -0
  119. provekit/templates/flows/customer-support-translate-reply.yaml +58 -0
  120. provekit/templates/flows/customer-support-triage-route.yaml +73 -0
  121. provekit/templates/flows/developer-tooling-action-items.yaml +43 -0
  122. provekit/templates/flows/developer-tooling-answer.yaml +43 -0
  123. provekit/templates/flows/developer-tooling-classify-intent.yaml +43 -0
  124. provekit/templates/flows/developer-tooling-draft-critique-revise.yaml +75 -0
  125. provekit/templates/flows/developer-tooling-extract-to-json.yaml +43 -0
  126. provekit/templates/flows/developer-tooling-faq-match-answer.yaml +60 -0
  127. provekit/templates/flows/developer-tooling-keyword-extract.yaml +43 -0
  128. provekit/templates/flows/developer-tooling-language-route.yaml +103 -0
  129. provekit/templates/flows/developer-tooling-moderation-gate.yaml +73 -0
  130. provekit/templates/flows/developer-tooling-redact-summarize.yaml +59 -0
  131. provekit/templates/flows/developer-tooling-score-gate.yaml +73 -0
  132. provekit/templates/flows/developer-tooling-sentiment-route.yaml +73 -0
  133. provekit/templates/flows/developer-tooling-severity-tiers.yaml +103 -0
  134. provekit/templates/flows/developer-tooling-summarize-then-decide.yaml +88 -0
  135. provekit/templates/flows/developer-tooling-tag-enrich.yaml +43 -0
  136. provekit/templates/flows/developer-tooling-tone-rewrite.yaml +43 -0
  137. provekit/templates/flows/developer-tooling-translate-reply.yaml +58 -0
  138. provekit/templates/flows/developer-tooling-triage-route.yaml +73 -0
  139. provekit/templates/flows/devrel-action-items.yaml +43 -0
  140. provekit/templates/flows/devrel-answer.yaml +43 -0
  141. provekit/templates/flows/devrel-classify-intent.yaml +43 -0
  142. provekit/templates/flows/devrel-draft-critique-revise.yaml +75 -0
  143. provekit/templates/flows/devrel-extract-to-json.yaml +43 -0
  144. provekit/templates/flows/devrel-faq-match-answer.yaml +60 -0
  145. provekit/templates/flows/devrel-keyword-extract.yaml +43 -0
  146. provekit/templates/flows/devrel-language-route.yaml +103 -0
  147. provekit/templates/flows/devrel-moderation-gate.yaml +73 -0
  148. provekit/templates/flows/devrel-redact-summarize.yaml +59 -0
  149. provekit/templates/flows/devrel-score-gate.yaml +73 -0
  150. provekit/templates/flows/devrel-sentiment-route.yaml +73 -0
  151. provekit/templates/flows/devrel-severity-tiers.yaml +103 -0
  152. provekit/templates/flows/devrel-summarize-then-decide.yaml +88 -0
  153. provekit/templates/flows/devrel-tag-enrich.yaml +43 -0
  154. provekit/templates/flows/devrel-tone-rewrite.yaml +43 -0
  155. provekit/templates/flows/devrel-translate-reply.yaml +58 -0
  156. provekit/templates/flows/devrel-triage-route.yaml +73 -0
  157. provekit/templates/flows/e-commerce-action-items.yaml +43 -0
  158. provekit/templates/flows/e-commerce-answer.yaml +43 -0
  159. provekit/templates/flows/e-commerce-classify-intent.yaml +43 -0
  160. provekit/templates/flows/e-commerce-draft-critique-revise.yaml +75 -0
  161. provekit/templates/flows/e-commerce-extract-to-json.yaml +43 -0
  162. provekit/templates/flows/e-commerce-faq-match-answer.yaml +60 -0
  163. provekit/templates/flows/e-commerce-keyword-extract.yaml +43 -0
  164. provekit/templates/flows/e-commerce-language-route.yaml +103 -0
  165. provekit/templates/flows/e-commerce-moderation-gate.yaml +73 -0
  166. provekit/templates/flows/e-commerce-redact-summarize.yaml +59 -0
  167. provekit/templates/flows/e-commerce-score-gate.yaml +73 -0
  168. provekit/templates/flows/e-commerce-sentiment-route.yaml +73 -0
  169. provekit/templates/flows/e-commerce-severity-tiers.yaml +103 -0
  170. provekit/templates/flows/e-commerce-summarize-then-decide.yaml +88 -0
  171. provekit/templates/flows/e-commerce-tag-enrich.yaml +43 -0
  172. provekit/templates/flows/e-commerce-tone-rewrite.yaml +43 -0
  173. provekit/templates/flows/e-commerce-translate-reply.yaml +58 -0
  174. provekit/templates/flows/e-commerce-triage-route.yaml +73 -0
  175. provekit/templates/flows/education-action-items.yaml +43 -0
  176. provekit/templates/flows/education-answer.yaml +43 -0
  177. provekit/templates/flows/education-classify-intent.yaml +43 -0
  178. provekit/templates/flows/education-draft-critique-revise.yaml +75 -0
  179. provekit/templates/flows/education-extract-to-json.yaml +43 -0
  180. provekit/templates/flows/education-faq-match-answer.yaml +60 -0
  181. provekit/templates/flows/education-keyword-extract.yaml +43 -0
  182. provekit/templates/flows/education-language-route.yaml +103 -0
  183. provekit/templates/flows/education-moderation-gate.yaml +73 -0
  184. provekit/templates/flows/education-redact-summarize.yaml +59 -0
  185. provekit/templates/flows/education-score-gate.yaml +73 -0
  186. provekit/templates/flows/education-sentiment-route.yaml +73 -0
  187. provekit/templates/flows/education-severity-tiers.yaml +103 -0
  188. provekit/templates/flows/education-summarize-then-decide.yaml +88 -0
  189. provekit/templates/flows/education-tag-enrich.yaml +43 -0
  190. provekit/templates/flows/education-tone-rewrite.yaml +43 -0
  191. provekit/templates/flows/education-translate-reply.yaml +58 -0
  192. provekit/templates/flows/education-triage-route.yaml +73 -0
  193. provekit/templates/flows/finance-action-items.yaml +43 -0
  194. provekit/templates/flows/finance-answer.yaml +43 -0
  195. provekit/templates/flows/finance-classify-intent.yaml +43 -0
  196. provekit/templates/flows/finance-draft-critique-revise.yaml +75 -0
  197. provekit/templates/flows/finance-extract-to-json.yaml +43 -0
  198. provekit/templates/flows/finance-faq-match-answer.yaml +60 -0
  199. provekit/templates/flows/finance-keyword-extract.yaml +43 -0
  200. provekit/templates/flows/finance-language-route.yaml +103 -0
  201. provekit/templates/flows/finance-moderation-gate.yaml +73 -0
  202. provekit/templates/flows/finance-redact-summarize.yaml +59 -0
  203. provekit/templates/flows/finance-score-gate.yaml +73 -0
  204. provekit/templates/flows/finance-sentiment-route.yaml +73 -0
  205. provekit/templates/flows/finance-severity-tiers.yaml +103 -0
  206. provekit/templates/flows/finance-summarize-then-decide.yaml +88 -0
  207. provekit/templates/flows/finance-tag-enrich.yaml +43 -0
  208. provekit/templates/flows/finance-tone-rewrite.yaml +43 -0
  209. provekit/templates/flows/finance-translate-reply.yaml +58 -0
  210. provekit/templates/flows/finance-triage-route.yaml +73 -0
  211. provekit/templates/flows/food-delivery-action-items.yaml +43 -0
  212. provekit/templates/flows/food-delivery-answer.yaml +43 -0
  213. provekit/templates/flows/food-delivery-classify-intent.yaml +43 -0
  214. provekit/templates/flows/food-delivery-draft-critique-revise.yaml +75 -0
  215. provekit/templates/flows/food-delivery-extract-to-json.yaml +43 -0
  216. provekit/templates/flows/food-delivery-faq-match-answer.yaml +60 -0
  217. provekit/templates/flows/food-delivery-keyword-extract.yaml +43 -0
  218. provekit/templates/flows/food-delivery-language-route.yaml +103 -0
  219. provekit/templates/flows/food-delivery-moderation-gate.yaml +73 -0
  220. provekit/templates/flows/food-delivery-redact-summarize.yaml +59 -0
  221. provekit/templates/flows/food-delivery-score-gate.yaml +73 -0
  222. provekit/templates/flows/food-delivery-sentiment-route.yaml +73 -0
  223. provekit/templates/flows/food-delivery-severity-tiers.yaml +103 -0
  224. provekit/templates/flows/food-delivery-summarize-then-decide.yaml +88 -0
  225. provekit/templates/flows/food-delivery-tag-enrich.yaml +43 -0
  226. provekit/templates/flows/food-delivery-tone-rewrite.yaml +43 -0
  227. provekit/templates/flows/food-delivery-translate-reply.yaml +58 -0
  228. provekit/templates/flows/food-delivery-triage-route.yaml +73 -0
  229. provekit/templates/flows/gaming-support-action-items.yaml +43 -0
  230. provekit/templates/flows/gaming-support-answer.yaml +43 -0
  231. provekit/templates/flows/gaming-support-classify-intent.yaml +43 -0
  232. provekit/templates/flows/gaming-support-draft-critique-revise.yaml +75 -0
  233. provekit/templates/flows/gaming-support-extract-to-json.yaml +43 -0
  234. provekit/templates/flows/gaming-support-faq-match-answer.yaml +60 -0
  235. provekit/templates/flows/gaming-support-keyword-extract.yaml +43 -0
  236. provekit/templates/flows/gaming-support-language-route.yaml +103 -0
  237. provekit/templates/flows/gaming-support-moderation-gate.yaml +73 -0
  238. provekit/templates/flows/gaming-support-redact-summarize.yaml +59 -0
  239. provekit/templates/flows/gaming-support-score-gate.yaml +73 -0
  240. provekit/templates/flows/gaming-support-sentiment-route.yaml +73 -0
  241. provekit/templates/flows/gaming-support-severity-tiers.yaml +103 -0
  242. provekit/templates/flows/gaming-support-summarize-then-decide.yaml +88 -0
  243. provekit/templates/flows/gaming-support-tag-enrich.yaml +43 -0
  244. provekit/templates/flows/gaming-support-tone-rewrite.yaml +43 -0
  245. provekit/templates/flows/gaming-support-translate-reply.yaml +58 -0
  246. provekit/templates/flows/gaming-support-triage-route.yaml +73 -0
  247. provekit/templates/flows/healthcare-intake-action-items.yaml +43 -0
  248. provekit/templates/flows/healthcare-intake-answer.yaml +43 -0
  249. provekit/templates/flows/healthcare-intake-classify-intent.yaml +43 -0
  250. provekit/templates/flows/healthcare-intake-draft-critique-revise.yaml +75 -0
  251. provekit/templates/flows/healthcare-intake-extract-to-json.yaml +43 -0
  252. provekit/templates/flows/healthcare-intake-faq-match-answer.yaml +60 -0
  253. provekit/templates/flows/healthcare-intake-keyword-extract.yaml +43 -0
  254. provekit/templates/flows/healthcare-intake-language-route.yaml +103 -0
  255. provekit/templates/flows/healthcare-intake-moderation-gate.yaml +73 -0
  256. provekit/templates/flows/healthcare-intake-redact-summarize.yaml +59 -0
  257. provekit/templates/flows/healthcare-intake-score-gate.yaml +73 -0
  258. provekit/templates/flows/healthcare-intake-sentiment-route.yaml +73 -0
  259. provekit/templates/flows/healthcare-intake-severity-tiers.yaml +103 -0
  260. provekit/templates/flows/healthcare-intake-summarize-then-decide.yaml +88 -0
  261. provekit/templates/flows/healthcare-intake-tag-enrich.yaml +43 -0
  262. provekit/templates/flows/healthcare-intake-tone-rewrite.yaml +43 -0
  263. provekit/templates/flows/healthcare-intake-translate-reply.yaml +58 -0
  264. provekit/templates/flows/healthcare-intake-triage-route.yaml +73 -0
  265. provekit/templates/flows/hr-action-items.yaml +43 -0
  266. provekit/templates/flows/hr-answer.yaml +43 -0
  267. provekit/templates/flows/hr-classify-intent.yaml +43 -0
  268. provekit/templates/flows/hr-draft-critique-revise.yaml +75 -0
  269. provekit/templates/flows/hr-extract-to-json.yaml +43 -0
  270. provekit/templates/flows/hr-faq-match-answer.yaml +60 -0
  271. provekit/templates/flows/hr-keyword-extract.yaml +43 -0
  272. provekit/templates/flows/hr-language-route.yaml +103 -0
  273. provekit/templates/flows/hr-moderation-gate.yaml +73 -0
  274. provekit/templates/flows/hr-redact-summarize.yaml +59 -0
  275. provekit/templates/flows/hr-score-gate.yaml +73 -0
  276. provekit/templates/flows/hr-sentiment-route.yaml +73 -0
  277. provekit/templates/flows/hr-severity-tiers.yaml +103 -0
  278. provekit/templates/flows/hr-summarize-then-decide.yaml +88 -0
  279. provekit/templates/flows/hr-tag-enrich.yaml +43 -0
  280. provekit/templates/flows/hr-tone-rewrite.yaml +43 -0
  281. provekit/templates/flows/hr-translate-reply.yaml +58 -0
  282. provekit/templates/flows/hr-triage-route.yaml +73 -0
  283. provekit/templates/flows/insurance-action-items.yaml +43 -0
  284. provekit/templates/flows/insurance-answer.yaml +43 -0
  285. provekit/templates/flows/insurance-classify-intent.yaml +43 -0
  286. provekit/templates/flows/insurance-draft-critique-revise.yaml +75 -0
  287. provekit/templates/flows/insurance-extract-to-json.yaml +43 -0
  288. provekit/templates/flows/insurance-faq-match-answer.yaml +60 -0
  289. provekit/templates/flows/insurance-keyword-extract.yaml +43 -0
  290. provekit/templates/flows/insurance-language-route.yaml +103 -0
  291. provekit/templates/flows/insurance-moderation-gate.yaml +73 -0
  292. provekit/templates/flows/insurance-redact-summarize.yaml +59 -0
  293. provekit/templates/flows/insurance-score-gate.yaml +73 -0
  294. provekit/templates/flows/insurance-sentiment-route.yaml +73 -0
  295. provekit/templates/flows/insurance-severity-tiers.yaml +103 -0
  296. provekit/templates/flows/insurance-summarize-then-decide.yaml +88 -0
  297. provekit/templates/flows/insurance-tag-enrich.yaml +43 -0
  298. provekit/templates/flows/insurance-tone-rewrite.yaml +43 -0
  299. provekit/templates/flows/insurance-translate-reply.yaml +58 -0
  300. provekit/templates/flows/insurance-triage-route.yaml +73 -0
  301. provekit/templates/flows/it-helpdesk-action-items.yaml +43 -0
  302. provekit/templates/flows/it-helpdesk-answer.yaml +43 -0
  303. provekit/templates/flows/it-helpdesk-classify-intent.yaml +43 -0
  304. provekit/templates/flows/it-helpdesk-draft-critique-revise.yaml +75 -0
  305. provekit/templates/flows/it-helpdesk-extract-to-json.yaml +43 -0
  306. provekit/templates/flows/it-helpdesk-faq-match-answer.yaml +60 -0
  307. provekit/templates/flows/it-helpdesk-keyword-extract.yaml +43 -0
  308. provekit/templates/flows/it-helpdesk-language-route.yaml +103 -0
  309. provekit/templates/flows/it-helpdesk-moderation-gate.yaml +73 -0
  310. provekit/templates/flows/it-helpdesk-redact-summarize.yaml +59 -0
  311. provekit/templates/flows/it-helpdesk-score-gate.yaml +73 -0
  312. provekit/templates/flows/it-helpdesk-sentiment-route.yaml +73 -0
  313. provekit/templates/flows/it-helpdesk-severity-tiers.yaml +103 -0
  314. provekit/templates/flows/it-helpdesk-summarize-then-decide.yaml +88 -0
  315. provekit/templates/flows/it-helpdesk-tag-enrich.yaml +43 -0
  316. provekit/templates/flows/it-helpdesk-tone-rewrite.yaml +43 -0
  317. provekit/templates/flows/it-helpdesk-translate-reply.yaml +58 -0
  318. provekit/templates/flows/it-helpdesk-triage-route.yaml +73 -0
  319. provekit/templates/flows/legal-intake-action-items.yaml +43 -0
  320. provekit/templates/flows/legal-intake-answer.yaml +43 -0
  321. provekit/templates/flows/legal-intake-classify-intent.yaml +43 -0
  322. provekit/templates/flows/legal-intake-draft-critique-revise.yaml +75 -0
  323. provekit/templates/flows/legal-intake-extract-to-json.yaml +43 -0
  324. provekit/templates/flows/legal-intake-faq-match-answer.yaml +60 -0
  325. provekit/templates/flows/legal-intake-keyword-extract.yaml +43 -0
  326. provekit/templates/flows/legal-intake-language-route.yaml +103 -0
  327. provekit/templates/flows/legal-intake-moderation-gate.yaml +73 -0
  328. provekit/templates/flows/legal-intake-redact-summarize.yaml +59 -0
  329. provekit/templates/flows/legal-intake-score-gate.yaml +73 -0
  330. provekit/templates/flows/legal-intake-sentiment-route.yaml +73 -0
  331. provekit/templates/flows/legal-intake-severity-tiers.yaml +103 -0
  332. provekit/templates/flows/legal-intake-summarize-then-decide.yaml +88 -0
  333. provekit/templates/flows/legal-intake-tag-enrich.yaml +43 -0
  334. provekit/templates/flows/legal-intake-tone-rewrite.yaml +43 -0
  335. provekit/templates/flows/legal-intake-translate-reply.yaml +58 -0
  336. provekit/templates/flows/legal-intake-triage-route.yaml +73 -0
  337. provekit/templates/flows/logistics-action-items.yaml +43 -0
  338. provekit/templates/flows/logistics-answer.yaml +43 -0
  339. provekit/templates/flows/logistics-classify-intent.yaml +43 -0
  340. provekit/templates/flows/logistics-draft-critique-revise.yaml +75 -0
  341. provekit/templates/flows/logistics-extract-to-json.yaml +43 -0
  342. provekit/templates/flows/logistics-faq-match-answer.yaml +60 -0
  343. provekit/templates/flows/logistics-keyword-extract.yaml +43 -0
  344. provekit/templates/flows/logistics-language-route.yaml +103 -0
  345. provekit/templates/flows/logistics-moderation-gate.yaml +73 -0
  346. provekit/templates/flows/logistics-redact-summarize.yaml +59 -0
  347. provekit/templates/flows/logistics-score-gate.yaml +73 -0
  348. provekit/templates/flows/logistics-sentiment-route.yaml +73 -0
  349. provekit/templates/flows/logistics-severity-tiers.yaml +103 -0
  350. provekit/templates/flows/logistics-summarize-then-decide.yaml +88 -0
  351. provekit/templates/flows/logistics-tag-enrich.yaml +43 -0
  352. provekit/templates/flows/logistics-tone-rewrite.yaml +43 -0
  353. provekit/templates/flows/logistics-translate-reply.yaml +58 -0
  354. provekit/templates/flows/logistics-triage-route.yaml +73 -0
  355. provekit/templates/flows/marketing-action-items.yaml +43 -0
  356. provekit/templates/flows/marketing-answer.yaml +43 -0
  357. provekit/templates/flows/marketing-classify-intent.yaml +43 -0
  358. provekit/templates/flows/marketing-draft-critique-revise.yaml +75 -0
  359. provekit/templates/flows/marketing-extract-to-json.yaml +43 -0
  360. provekit/templates/flows/marketing-faq-match-answer.yaml +60 -0
  361. provekit/templates/flows/marketing-keyword-extract.yaml +43 -0
  362. provekit/templates/flows/marketing-language-route.yaml +103 -0
  363. provekit/templates/flows/marketing-moderation-gate.yaml +73 -0
  364. provekit/templates/flows/marketing-redact-summarize.yaml +59 -0
  365. provekit/templates/flows/marketing-score-gate.yaml +73 -0
  366. provekit/templates/flows/marketing-sentiment-route.yaml +73 -0
  367. provekit/templates/flows/marketing-severity-tiers.yaml +103 -0
  368. provekit/templates/flows/marketing-summarize-then-decide.yaml +88 -0
  369. provekit/templates/flows/marketing-tag-enrich.yaml +43 -0
  370. provekit/templates/flows/marketing-tone-rewrite.yaml +43 -0
  371. provekit/templates/flows/marketing-translate-reply.yaml +58 -0
  372. provekit/templates/flows/marketing-triage-route.yaml +73 -0
  373. provekit/templates/flows/nonprofit-action-items.yaml +43 -0
  374. provekit/templates/flows/nonprofit-answer.yaml +43 -0
  375. provekit/templates/flows/nonprofit-classify-intent.yaml +43 -0
  376. provekit/templates/flows/nonprofit-draft-critique-revise.yaml +75 -0
  377. provekit/templates/flows/nonprofit-extract-to-json.yaml +43 -0
  378. provekit/templates/flows/nonprofit-faq-match-answer.yaml +60 -0
  379. provekit/templates/flows/nonprofit-keyword-extract.yaml +43 -0
  380. provekit/templates/flows/nonprofit-language-route.yaml +103 -0
  381. provekit/templates/flows/nonprofit-moderation-gate.yaml +73 -0
  382. provekit/templates/flows/nonprofit-redact-summarize.yaml +59 -0
  383. provekit/templates/flows/nonprofit-score-gate.yaml +73 -0
  384. provekit/templates/flows/nonprofit-sentiment-route.yaml +73 -0
  385. provekit/templates/flows/nonprofit-severity-tiers.yaml +103 -0
  386. provekit/templates/flows/nonprofit-summarize-then-decide.yaml +88 -0
  387. provekit/templates/flows/nonprofit-tag-enrich.yaml +43 -0
  388. provekit/templates/flows/nonprofit-tone-rewrite.yaml +43 -0
  389. provekit/templates/flows/nonprofit-translate-reply.yaml +58 -0
  390. provekit/templates/flows/nonprofit-triage-route.yaml +73 -0
  391. provekit/templates/flows/real-estate-action-items.yaml +43 -0
  392. provekit/templates/flows/real-estate-answer.yaml +43 -0
  393. provekit/templates/flows/real-estate-classify-intent.yaml +43 -0
  394. provekit/templates/flows/real-estate-draft-critique-revise.yaml +75 -0
  395. provekit/templates/flows/real-estate-extract-to-json.yaml +43 -0
  396. provekit/templates/flows/real-estate-faq-match-answer.yaml +60 -0
  397. provekit/templates/flows/real-estate-keyword-extract.yaml +43 -0
  398. provekit/templates/flows/real-estate-language-route.yaml +103 -0
  399. provekit/templates/flows/real-estate-moderation-gate.yaml +73 -0
  400. provekit/templates/flows/real-estate-redact-summarize.yaml +59 -0
  401. provekit/templates/flows/real-estate-score-gate.yaml +73 -0
  402. provekit/templates/flows/real-estate-sentiment-route.yaml +73 -0
  403. provekit/templates/flows/real-estate-severity-tiers.yaml +103 -0
  404. provekit/templates/flows/real-estate-summarize-then-decide.yaml +88 -0
  405. provekit/templates/flows/real-estate-tag-enrich.yaml +43 -0
  406. provekit/templates/flows/real-estate-tone-rewrite.yaml +43 -0
  407. provekit/templates/flows/real-estate-translate-reply.yaml +58 -0
  408. provekit/templates/flows/real-estate-triage-route.yaml +73 -0
  409. provekit/templates/flows/recruiting-action-items.yaml +43 -0
  410. provekit/templates/flows/recruiting-answer.yaml +43 -0
  411. provekit/templates/flows/recruiting-classify-intent.yaml +43 -0
  412. provekit/templates/flows/recruiting-draft-critique-revise.yaml +75 -0
  413. provekit/templates/flows/recruiting-extract-to-json.yaml +43 -0
  414. provekit/templates/flows/recruiting-faq-match-answer.yaml +60 -0
  415. provekit/templates/flows/recruiting-keyword-extract.yaml +43 -0
  416. provekit/templates/flows/recruiting-language-route.yaml +103 -0
  417. provekit/templates/flows/recruiting-moderation-gate.yaml +73 -0
  418. provekit/templates/flows/recruiting-redact-summarize.yaml +59 -0
  419. provekit/templates/flows/recruiting-score-gate.yaml +73 -0
  420. provekit/templates/flows/recruiting-sentiment-route.yaml +73 -0
  421. provekit/templates/flows/recruiting-severity-tiers.yaml +103 -0
  422. provekit/templates/flows/recruiting-summarize-then-decide.yaml +88 -0
  423. provekit/templates/flows/recruiting-tag-enrich.yaml +43 -0
  424. provekit/templates/flows/recruiting-tone-rewrite.yaml +43 -0
  425. provekit/templates/flows/recruiting-translate-reply.yaml +58 -0
  426. provekit/templates/flows/recruiting-triage-route.yaml +73 -0
  427. provekit/templates/flows/research-action-items.yaml +43 -0
  428. provekit/templates/flows/research-answer.yaml +43 -0
  429. provekit/templates/flows/research-classify-intent.yaml +43 -0
  430. provekit/templates/flows/research-draft-critique-revise.yaml +75 -0
  431. provekit/templates/flows/research-extract-to-json.yaml +43 -0
  432. provekit/templates/flows/research-faq-match-answer.yaml +60 -0
  433. provekit/templates/flows/research-keyword-extract.yaml +43 -0
  434. provekit/templates/flows/research-language-route.yaml +103 -0
  435. provekit/templates/flows/research-moderation-gate.yaml +73 -0
  436. provekit/templates/flows/research-redact-summarize.yaml +59 -0
  437. provekit/templates/flows/research-score-gate.yaml +73 -0
  438. provekit/templates/flows/research-sentiment-route.yaml +73 -0
  439. provekit/templates/flows/research-severity-tiers.yaml +103 -0
  440. provekit/templates/flows/research-summarize-then-decide.yaml +88 -0
  441. provekit/templates/flows/research-tag-enrich.yaml +43 -0
  442. provekit/templates/flows/research-tone-rewrite.yaml +43 -0
  443. provekit/templates/flows/research-translate-reply.yaml +58 -0
  444. provekit/templates/flows/research-triage-route.yaml +73 -0
  445. provekit/templates/flows/sales-lead-action-items.yaml +43 -0
  446. provekit/templates/flows/sales-lead-answer.yaml +43 -0
  447. provekit/templates/flows/sales-lead-classify-intent.yaml +43 -0
  448. provekit/templates/flows/sales-lead-draft-critique-revise.yaml +75 -0
  449. provekit/templates/flows/sales-lead-extract-to-json.yaml +43 -0
  450. provekit/templates/flows/sales-lead-faq-match-answer.yaml +60 -0
  451. provekit/templates/flows/sales-lead-keyword-extract.yaml +43 -0
  452. provekit/templates/flows/sales-lead-language-route.yaml +103 -0
  453. provekit/templates/flows/sales-lead-moderation-gate.yaml +73 -0
  454. provekit/templates/flows/sales-lead-redact-summarize.yaml +59 -0
  455. provekit/templates/flows/sales-lead-score-gate.yaml +73 -0
  456. provekit/templates/flows/sales-lead-sentiment-route.yaml +73 -0
  457. provekit/templates/flows/sales-lead-severity-tiers.yaml +103 -0
  458. provekit/templates/flows/sales-lead-summarize-then-decide.yaml +88 -0
  459. provekit/templates/flows/sales-lead-tag-enrich.yaml +43 -0
  460. provekit/templates/flows/sales-lead-tone-rewrite.yaml +43 -0
  461. provekit/templates/flows/sales-lead-translate-reply.yaml +58 -0
  462. provekit/templates/flows/sales-lead-triage-route.yaml +73 -0
  463. provekit/templates/flows/travel-action-items.yaml +43 -0
  464. provekit/templates/flows/travel-answer.yaml +43 -0
  465. provekit/templates/flows/travel-classify-intent.yaml +43 -0
  466. provekit/templates/flows/travel-draft-critique-revise.yaml +75 -0
  467. provekit/templates/flows/travel-extract-to-json.yaml +43 -0
  468. provekit/templates/flows/travel-faq-match-answer.yaml +60 -0
  469. provekit/templates/flows/travel-keyword-extract.yaml +43 -0
  470. provekit/templates/flows/travel-language-route.yaml +103 -0
  471. provekit/templates/flows/travel-moderation-gate.yaml +73 -0
  472. provekit/templates/flows/travel-redact-summarize.yaml +59 -0
  473. provekit/templates/flows/travel-score-gate.yaml +73 -0
  474. provekit/templates/flows/travel-sentiment-route.yaml +73 -0
  475. provekit/templates/flows/travel-severity-tiers.yaml +103 -0
  476. provekit/templates/flows/travel-summarize-then-decide.yaml +88 -0
  477. provekit/templates/flows/travel-tag-enrich.yaml +43 -0
  478. provekit/templates/flows/travel-tone-rewrite.yaml +43 -0
  479. provekit/templates/flows/travel-translate-reply.yaml +58 -0
  480. provekit/templates/flows/travel-triage-route.yaml +73 -0
  481. provekit-0.1.0.dist-info/METADATA +86 -0
  482. provekit-0.1.0.dist-info/RECORD +485 -0
  483. provekit-0.1.0.dist-info/WHEEL +5 -0
  484. provekit-0.1.0.dist-info/entry_points.txt +2 -0
  485. provekit-0.1.0.dist-info/top_level.txt +1 -0
provekit/__init__.py ADDED
File without changes
provekit/cli.py ADDED
@@ -0,0 +1,256 @@
1
+ """`provekit` — headless runner for .provekit test files (CI).
2
+
3
+ provekit run .provekit/tests/ # run every test file under a dir
4
+ provekit run support-bot.yaml --format junit -o results.xml
5
+ provekit import-promptfoo promptfooconfig.yaml -o .provekit/tests/
6
+
7
+ Connections resolve from a local file (default: .provekit/connections.yaml, then
8
+ ~/.provekit/connections.yaml), NEVER from the test files. Secrets in that file may
9
+ be written as ${ENV_VAR} and are expanded from the environment at run time.
10
+
11
+ Exit code is non-zero when any assertion fails or any run errors — so CI gates on it.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import os
17
+ import re
18
+ import sys
19
+ from pathlib import Path
20
+ from xml.sax.saxutils import escape
21
+
22
+ import yaml
23
+
24
+ from .services import assertions as ae
25
+ from .services import dispatch, testfile
26
+ from .services.promptfoo import import_promptfoo
27
+
28
+ _ENV_REF = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
29
+
30
+
31
+ # ---- local connection registry (stands in for the DB in CLI mode) ----
32
+ class _Conn:
33
+ def __init__(self, config):
34
+ self.config = config
35
+
36
+
37
+ class _Registry:
38
+ """Minimal object graph dispatch/assertions expect from a `db`: `.get(model, id)`."""
39
+ def __init__(self, by_name: dict):
40
+ self._by_name = by_name
41
+ self._ids = {name: i + 1 for i, name in enumerate(by_name)}
42
+ self._by_id = {i: _Conn(by_name[name]) for name, i in self._ids.items()}
43
+
44
+ def id_of(self, name: str) -> int | None:
45
+ return self._ids.get(name)
46
+
47
+ def get(self, _model, cid):
48
+ return self._by_id.get(cid)
49
+
50
+
51
+ def _expand_env(obj):
52
+ if isinstance(obj, str):
53
+ return _ENV_REF.sub(lambda m: os.environ.get(m.group(1), ""), obj)
54
+ if isinstance(obj, dict):
55
+ return {k: _expand_env(v) for k, v in obj.items()}
56
+ if isinstance(obj, list):
57
+ return [_expand_env(v) for v in obj]
58
+ return obj
59
+
60
+
61
+ def _load_connections(explicit: str | None) -> _Registry:
62
+ paths = [explicit] if explicit else [".provekit/connections.yaml",
63
+ str(Path.home() / ".provekit/connections.yaml")]
64
+ for p in paths:
65
+ if p and Path(p).exists():
66
+ doc = yaml.safe_load(Path(p).read_text()) or {}
67
+ return _Registry(_expand_env(doc.get("connections") or {}))
68
+ return _Registry({})
69
+
70
+
71
+ # ---- running ----
72
+ def _run_case(reg, request, variables):
73
+ rd = _collect(reg, request, variables)
74
+ asserts = ae.evaluate(reg, request.get("assertions") or [], rd)
75
+ ok = all(a["ok"] for a in asserts) if asserts else rd["status"] == "completed"
76
+ return {"status": rd["status"], "error": rd["error"], "duration_ms": rd["duration_ms"],
77
+ "text": rd["result"].get("text"), "assertions": asserts, "ok": ok}
78
+
79
+
80
+ def _collect(reg, req, variables):
81
+ # dispatch is async; the CLI is sync, so drive it via the sync bridge (its own loop).
82
+ r = dispatch.run_collect_sync(reg, req, variables)
83
+ return {"result": {"text": r["text"], "output": r["output"], "meta": r["meta"]},
84
+ "status": r["status"], "error": r["error"], "duration_ms": r.get("duration_ms", 0)}
85
+
86
+
87
+ def _resolve_request(doc, reg) -> tuple[dict, str | None]:
88
+ """Build the runnable request from a test doc; returns (request, unresolved_connection)."""
89
+ req = dict(doc["request"])
90
+ name = doc.get("connection")
91
+ cid = reg.id_of(name) if name else None
92
+ if cid:
93
+ req["connection_id"] = cid
94
+ if doc.get("assertions"):
95
+ req["assertions"] = doc["assertions"]
96
+ # llm_judge assertions may name their own judge connection
97
+ for a in req.get("assertions") or []:
98
+ if a.get("type") == "llm_judge" and a.get("connection"):
99
+ a["connection_id"] = reg.id_of(a["connection"])
100
+ return req, (name if name and not cid else None)
101
+
102
+
103
+ def _iter_test_files(paths):
104
+ for p in paths:
105
+ path = Path(p)
106
+ if path.is_dir():
107
+ yield from sorted(path.rglob("*.yaml"))
108
+ yield from sorted(path.rglob("*.yml"))
109
+ elif path.exists():
110
+ yield path
111
+ else:
112
+ print(f"warning: path not found: {p}", file=sys.stderr)
113
+
114
+
115
+ def cmd_run(args) -> int:
116
+ reg = _load_connections(args.connections)
117
+ cli_vars = dict(kv.split("=", 1) for kv in args.env) if args.env else {}
118
+ files = list(_iter_test_files(args.paths))
119
+ if not files:
120
+ print("no .provekit test files found", file=sys.stderr)
121
+ return 2
122
+
123
+ suites = []
124
+ for f in files:
125
+ try:
126
+ doc = testfile.load(f.read_text())
127
+ except ValueError as exc:
128
+ suites.append({"file": str(f), "error": f"parse error: {exc}", "cases": []})
129
+ continue
130
+ if doc.get("kind") != "test":
131
+ continue # flows aren't CI test units
132
+ request, unresolved = _resolve_request(doc, reg)
133
+ rows = doc.get("dataset") or [{"name": None, "variables": {}}]
134
+ cases = []
135
+ for i, row in enumerate(rows):
136
+ variables = {**cli_vars, **(row.get("variables") or {})}
137
+ res = _run_case(reg, request, variables)
138
+ res["name"] = row.get("name") or (f"row {i + 1}" if len(rows) > 1 else doc["name"])
139
+ cases.append(res)
140
+ suites.append({"file": str(f), "name": doc["name"], "cases": cases,
141
+ "error": (f"connection '{unresolved}' not found" if unresolved else None)})
142
+
143
+ _render(suites, args.format, args.output)
144
+ failed = sum(1 for s in suites for c in s["cases"] if not c["ok"]) + \
145
+ sum(1 for s in suites if s.get("error"))
146
+ return 1 if failed else 0
147
+
148
+
149
+ # ---- output renderers ----
150
+ _G, _R, _Y, _D, _X = "\033[32m", "\033[31m", "\033[33m", "\033[2m", "\033[0m"
151
+
152
+
153
+ def _color(on: bool):
154
+ return ("", "", "", "", "") if not on else (_G, _R, _Y, _D, _X)
155
+
156
+
157
+ def _render(suites, fmt, output):
158
+ if fmt == "json":
159
+ import json
160
+ text = json.dumps({"suites": suites}, indent=2)
161
+ elif fmt == "junit":
162
+ text = _junit(suites)
163
+ else:
164
+ text = _pretty(suites, color=output is None and sys.stdout.isatty())
165
+ if output:
166
+ Path(output).write_text(text)
167
+ print(f"wrote {fmt} report to {output}", file=sys.stderr)
168
+ else:
169
+ print(text)
170
+
171
+
172
+ def _pretty(suites, color: bool) -> str:
173
+ g, r, y, d, x = _color(color)
174
+ lines, total, passed = [], 0, 0
175
+ for s in suites:
176
+ if s.get("error") and not s["cases"]:
177
+ lines.append(f"{r}✕{x} {s['file']} — {s['error']}")
178
+ continue
179
+ lines.append(f"{d}{s['file']}{x}")
180
+ if s.get("error"):
181
+ lines.append(f" {y}! {s['error']}{x}")
182
+ for c in s["cases"]:
183
+ total += 1; passed += c["ok"]
184
+ mark = f"{g}✓{x}" if c["ok"] else f"{r}✕{x}"
185
+ lines.append(f" {mark} {c['name']} {d}{c['duration_ms']}ms{x}")
186
+ for a in c["assertions"]:
187
+ am = f"{g}✓{x}" if a["ok"] else f"{r}✕{x}"
188
+ lines.append(f" {am} {a['type']}: {a['detail']}")
189
+ if c["error"]:
190
+ lines.append(f" {r}error: {c['error']}{x}")
191
+ head = f"{g if passed == total else r}{passed}/{total} passed{x}"
192
+ lines.append("")
193
+ lines.append(head)
194
+ return "\n".join(lines)
195
+
196
+
197
+ def _junit(suites) -> str:
198
+ out = ['<?xml version="1.0" encoding="UTF-8"?>', "<testsuites>"]
199
+ for s in suites:
200
+ cases = s["cases"]
201
+ fails = sum(1 for c in cases if not c["ok"])
202
+ errs = 1 if s.get("error") and not cases else 0
203
+ out.append(f' <testsuite name="{escape(s.get("name") or s["file"])}" '
204
+ f'tests="{len(cases)}" failures="{fails}" errors="{errs}">')
205
+ if errs:
206
+ out.append(f' <testcase name="{escape(s["file"])}">'
207
+ f'<error message="{escape(s["error"])}"/></testcase>')
208
+ for c in cases:
209
+ t = (c["duration_ms"] or 0) / 1000
210
+ out.append(f' <testcase name="{escape(c["name"])}" time="{t:.3f}">')
211
+ if not c["ok"]:
212
+ detail = c["error"] or "; ".join(f"{a['type']}: {a['detail']}"
213
+ for a in c["assertions"] if not a["ok"])
214
+ out.append(f' <failure message="{escape(detail)}"/>')
215
+ out.append(" </testcase>")
216
+ out.append(" </testsuite>")
217
+ out.append("</testsuites>")
218
+ return "\n".join(out)
219
+
220
+
221
+ def cmd_import_promptfoo(args) -> int:
222
+ docs, warnings = import_promptfoo(Path(args.config).read_text())
223
+ outdir = Path(args.output or ".")
224
+ outdir.mkdir(parents=True, exist_ok=True)
225
+ for name, text in docs:
226
+ (outdir / name).write_text(text)
227
+ print(f"wrote {outdir / name}")
228
+ for w in warnings: # never drop anything silently
229
+ print(f"warning: {w}", file=sys.stderr)
230
+ print(f"imported {len(docs)} test(s) from {args.config}", file=sys.stderr)
231
+ return 0
232
+
233
+
234
+ def main(argv=None) -> int:
235
+ parser = argparse.ArgumentParser(prog="provekit", description="Run .provekit agent tests.")
236
+ sub = parser.add_subparsers(dest="cmd", required=True)
237
+
238
+ run = sub.add_parser("run", help="run test files or directories")
239
+ run.add_argument("paths", nargs="+")
240
+ run.add_argument("--connections", help="connections YAML (default: .provekit/connections.yaml)")
241
+ run.add_argument("--env", action="append", metavar="KEY=VAL", help="extra template variables")
242
+ run.add_argument("--format", choices=["pretty", "json", "junit"], default="pretty")
243
+ run.add_argument("-o", "--output", help="write the report to a file instead of stdout")
244
+ run.set_defaults(func=cmd_run)
245
+
246
+ imp = sub.add_parser("import-promptfoo", help="convert a promptfoo config to .provekit tests")
247
+ imp.add_argument("config")
248
+ imp.add_argument("-o", "--output", help="output directory (default: cwd)")
249
+ imp.set_defaults(func=cmd_import_promptfoo)
250
+
251
+ args = parser.parse_args(argv)
252
+ return args.func(args)
253
+
254
+
255
+ if __name__ == "__main__":
256
+ sys.exit(main())
provekit/config.py ADDED
@@ -0,0 +1,77 @@
1
+ """ProveKit configuration."""
2
+ from functools import lru_cache
3
+
4
+ from pydantic_settings import BaseSettings, SettingsConfigDict
5
+
6
+
7
+ class Settings(BaseSettings):
8
+ model_config = SettingsConfigDict(env_file=".env", extra="ignore")
9
+
10
+ database_url: str = "sqlite:///./provekit.db"
11
+ cors_origins: str = "http://localhost:3001,http://localhost:3000"
12
+
13
+ # Hosted mode: stricter outbound-URL guarding (block private/internal ranges).
14
+ # Leave false for local single-user use.
15
+ hosted: bool = False
16
+
17
+ # Encryption key for secrets at rest (any string; derived to a Fernet key).
18
+ # Optional for local SQLite (a key file is auto-generated next to the db);
19
+ # required for any other database.
20
+ secret_key: str = ""
21
+
22
+ # Optional OTLP/HTTP collector to mirror ProveKit's own runs as gen_ai spans.
23
+ otel_export_url: str = ""
24
+
25
+ # Public base URL shown in deployment endpoints (where /v1/d/{slug} is reachable).
26
+ public_base_url: str = "http://localhost:8100"
27
+
28
+ # Wall-clock cap for a single deployment invocation (seconds); 0 disables.
29
+ deployment_timeout_s: float = 120.0
30
+
31
+ # Optional Sentry DSN for error reporting (off when unset).
32
+ sentry_dsn: str = ""
33
+
34
+ # Web app base URL (for links in emails). Same-origin in prod.
35
+ web_base_url: str = "http://localhost:3001"
36
+
37
+ # Email (account recovery/verification). No SMTP host → links are logged, not sent.
38
+ smtp_host: str = ""
39
+ smtp_port: int = 587
40
+ smtp_user: str = ""
41
+ smtp_password: str = ""
42
+ smtp_from: str = ""
43
+ smtp_starttls: bool = True
44
+ require_email_verification: bool = False
45
+
46
+ # Optional Redis for paused flow-run contexts (survives workers/restarts).
47
+ # Unset → in-memory (single-process local use).
48
+ redis_url: str = ""
49
+
50
+ # Max concurrent sync workers (streams run in this threadpool). Lifts Starlette's
51
+ # ~40-token default so many more streams run at once without an async rewrite.
52
+ thread_pool_size: int = 200
53
+
54
+ # Quotas (protect shared infra). 0 disables a given limit.
55
+ rate_limit_per_min: int = 120 # run requests per workspace per minute
56
+ login_attempts_per_min: int = 10 # login attempts per email+IP per minute
57
+ dataset_max_rows: int = 200 # rows per dataset run
58
+ max_tokens_cap: int = 0 # hard cap on prompt max_tokens (0 = no cap)
59
+ runs_retention: int = 1000 # keep last N runs per workspace
60
+ max_body_bytes: int = 2_000_000 # reject request bodies larger than this
61
+ max_flow_nodes: int = 200 # cap nodes per flow
62
+
63
+ # Seed empty OpenAI + Anthropic example connections on first run. The keyless
64
+ # "Demo Assistant (mock)" connection is seeded regardless of this setting.
65
+ seed_examples: bool = True
66
+
67
+ # A default OpenAI key can be provided to prefill the example LLM connection.
68
+ openai_api_key: str = ""
69
+
70
+ @property
71
+ def cors_origin_list(self) -> list[str]:
72
+ return [o.strip() for o in self.cors_origins.split(",") if o.strip()]
73
+
74
+
75
+ @lru_cache
76
+ def get_settings() -> Settings:
77
+ return Settings()
provekit/database.py ADDED
@@ -0,0 +1,132 @@
1
+ """SQLAlchemy engine + session (SQLite)."""
2
+ from sqlalchemy import MetaData, create_engine, event
3
+ from sqlalchemy.orm import DeclarativeBase, sessionmaker
4
+
5
+ from .config import get_settings
6
+
7
+ settings = get_settings()
8
+ _is_sqlite = settings.database_url.startswith("sqlite")
9
+ connect_args = {"check_same_thread": False} if _is_sqlite else {}
10
+ # A concurrent stream holds a pooled connection for its lifetime; if the pool is exhausted,
11
+ # the next (synchronous) checkout blocks the event loop. Size the pool to the stream
12
+ # concurrency so checkouts never block the loop.
13
+ if _is_sqlite and ":memory:" in settings.database_url:
14
+ _pool = {}
15
+ else:
16
+ _pool = {"pool_size": 20, "max_overflow": max(10, settings.thread_pool_size)}
17
+ if not _is_sqlite:
18
+ _pool["pool_pre_ping"] = True # Postgres: drop stale connections (harmful w/ SQLite WAL pragma)
19
+ engine = create_engine(settings.database_url, connect_args=connect_args, **_pool)
20
+ SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
21
+
22
+ if _is_sqlite:
23
+ @event.listens_for(engine, "connect")
24
+ def _sqlite_pragmas(dbapi_conn, _record):
25
+ # WAL lets readers and a writer coexist; busy_timeout makes a contended write wait
26
+ # briefly instead of erroring — important now that many async streams write runs.
27
+ cur = dbapi_conn.cursor()
28
+ cur.execute("PRAGMA journal_mode=WAL")
29
+ cur.execute("PRAGMA busy_timeout=5000")
30
+ cur.close()
31
+
32
+ # Named constraints so Alembic's SQLite batch mode (ALTER via table-rebuild) can re-add them.
33
+ _NAMING = {
34
+ "ix": "ix_%(column_0_label)s",
35
+ "uq": "uq_%(table_name)s_%(column_0_name)s",
36
+ "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
37
+ "pk": "pk_%(table_name)s",
38
+ }
39
+
40
+
41
+ class Base(DeclarativeBase):
42
+ metadata = MetaData(naming_convention=_NAMING)
43
+
44
+
45
+ def get_db():
46
+ db = SessionLocal()
47
+ try:
48
+ yield db
49
+ finally:
50
+ db.close()
51
+
52
+
53
+ def init_db() -> None:
54
+ from . import models # noqa: F401
55
+ if settings.database_url.startswith("sqlite") and ":memory:" in settings.database_url:
56
+ # Ephemeral in-memory DB (tests only): create_all is fast and needs no migration tooling.
57
+ Base.metadata.create_all(bind=engine)
58
+ else:
59
+ # Every persistent DB — file-based SQLite included — is owned by Alembic so that
60
+ # future column/index changes actually apply on upgrade. create_all only ever adds
61
+ # whole tables and would silently skip new columns on an existing local database.
62
+ _run_migrations()
63
+
64
+
65
+ # Revision that first introduced each column, newest first — used to adopt a pre-migration
66
+ # database (built by the old create_all path) at the revision matching its actual columns.
67
+ _ADOPT_BY_COLUMN = [
68
+ ("users", "token_version", "a1b2c3d4e5f6"),
69
+ ("users", "email_verified", "323eb73d463c"),
70
+ ("workspaces", "ingest_key_hash", "819ed5ff183e"),
71
+ ]
72
+ _BASELINE_REVISION = "49e8ab812556"
73
+
74
+
75
+ # Postgres advisory-lock key ("AGMN") used to serialize migrations across uvicorn workers.
76
+ _MIGRATION_LOCK_ID = 0x4147_4D4E
77
+
78
+
79
+ def _migration_config():
80
+ """Alembic config built in code rather than read from alembic.ini.
81
+
82
+ The migrations ship *inside* the package, so script_location is resolved absolutely
83
+ from `__file__`. Reading alembic.ini instead would only work when the process runs
84
+ from a source checkout — a `pip install`ed wheel has no alembic.ini next to the
85
+ package, and the app would fail to boot. alembic.ini stays for `alembic` CLI use.
86
+ """
87
+ import os
88
+
89
+ from alembic.config import Config
90
+
91
+ cfg = Config()
92
+ cfg.set_main_option("script_location", os.path.join(os.path.dirname(__file__), "migrations"))
93
+ # No sqlalchemy.url here on purpose: env.py reads it from settings and builds the engine
94
+ # itself, which keeps a '%' in the password out of ConfigParser's interpolation.
95
+ return cfg
96
+
97
+
98
+ def _run_migrations() -> None:
99
+ cfg = _migration_config()
100
+ if settings.database_url.startswith("sqlite"):
101
+ _migrate_to_head(cfg) # single process — no cross-worker race
102
+ return
103
+ # With `--workers N` every worker runs the lifespan and would race through upgrade
104
+ # (DuplicateTable / duplicate alembic_version). A Postgres session advisory lock lets
105
+ # exactly one run at a time; the rest block, then find the schema already at head.
106
+ with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
107
+ conn.exec_driver_sql("SELECT pg_advisory_lock(%s)", (_MIGRATION_LOCK_ID,))
108
+ try:
109
+ _migrate_to_head(cfg)
110
+ finally:
111
+ conn.exec_driver_sql("SELECT pg_advisory_unlock(%s)", (_MIGRATION_LOCK_ID,))
112
+
113
+
114
+ def _migrate_to_head(cfg) -> None:
115
+ from alembic import command
116
+ from alembic.runtime.migration import MigrationContext
117
+ from sqlalchemy import inspect
118
+
119
+ insp = inspect(engine)
120
+ with engine.connect() as conn:
121
+ current = MigrationContext.configure(conn).get_current_revision()
122
+ if current is None and insp.has_table("users"):
123
+ # A database created by create_all before migrations existed has the tables but no
124
+ # alembic version. Stamp it at the revision matching its columns so `upgrade` applies
125
+ # only what's genuinely missing instead of trying to re-create existing tables.
126
+ stamp_at = _BASELINE_REVISION
127
+ for table, column, revision in _ADOPT_BY_COLUMN:
128
+ if any(c["name"] == column for c in insp.get_columns(table)):
129
+ stamp_at = revision
130
+ break
131
+ command.stamp(cfg, stamp_at)
132
+ command.upgrade(cfg, "head")
provekit/main.py ADDED
@@ -0,0 +1,115 @@
1
+ """ProveKit — a generic "Postman for agents": create, run, and debug prompts, MCP tools,
2
+ and agent endpoints across any provider."""
3
+ import logging
4
+ from contextlib import asynccontextmanager
5
+
6
+ from fastapi import FastAPI
7
+ from fastapi.middleware.cors import CORSMiddleware
8
+
9
+ from .config import get_settings
10
+ from .database import SessionLocal, init_db
11
+ from .models import Connection
12
+ from .observability import (
13
+ BodySizeLimitMiddleware,
14
+ RequestIDMiddleware,
15
+ SecurityHeadersMiddleware,
16
+ healthz,
17
+ init_sentry,
18
+ setup_logging,
19
+ )
20
+ from .routers import auth, connections, deployments, flows, library, prompts, run, runtime, traces, usage
21
+
22
+ logging.basicConfig(level=logging.INFO)
23
+ settings = get_settings()
24
+ setup_logging()
25
+ init_sentry()
26
+
27
+
28
+ def _reseal_connections(db) -> None:
29
+ """Upgrade plaintext secrets from older databases to encrypted-at-rest: force each
30
+ config back through SealedJSON's bind processor (flag_modified beats SQLAlchemy's
31
+ equal-value change suppression). Idempotent — already-sealed values stay sealed."""
32
+ from sqlalchemy.orm.attributes import flag_modified
33
+ for c in db.query(Connection).all():
34
+ flag_modified(c, "config")
35
+ db.commit()
36
+
37
+
38
+ _WEAK_KEYS = {"", "dev-only-change-me", "change-me", "changeme", "secret"}
39
+
40
+
41
+ def _guard_production_config() -> None:
42
+ """Refuse to boot hosted mode with an unset/weak SECRET_KEY — otherwise every stored
43
+ credential is decryptable by anyone who reads the (default, public) value."""
44
+ s = settings
45
+ weak_key = s.secret_key.strip() in _WEAK_KEYS or len(s.secret_key.strip()) < 16
46
+ if s.hosted and weak_key:
47
+ raise RuntimeError(
48
+ "HOSTED=true requires a strong SECRET_KEY (>=16 chars, not the dev default). "
49
+ "Generate one: python -c \"import secrets; print(secrets.token_urlsafe(48))\"")
50
+ # Not hosted but using a server database with a weak key = credentials encrypted under a
51
+ # public constant. Legitimate for the dev compose; warn loudly if it reaches a real server.
52
+ if not s.hosted and weak_key and not s.database_url.startswith("sqlite"):
53
+ logging.getLogger("provekit").warning(
54
+ "Weak/default SECRET_KEY with a non-SQLite database and HOSTED=false — stored "
55
+ "credentials are encrypted under a public constant. Set a strong SECRET_KEY (and "
56
+ "HOSTED=true) before exposing this to a network.")
57
+
58
+
59
+ @asynccontextmanager
60
+ async def lifespan(app: FastAPI):
61
+ _guard_production_config()
62
+ # Raise the threadpool ceiling: streaming responses run their sync generators here, so
63
+ # the default ~40 tokens caps concurrent streams. This lifts it without an async rewrite.
64
+ try:
65
+ import anyio
66
+ anyio.to_thread.current_default_thread_limiter().total_tokens = settings.thread_pool_size
67
+ except Exception:
68
+ pass
69
+ init_db()
70
+ db = SessionLocal()
71
+ try:
72
+ _reseal_connections(db) # per-workspace seeding happens on first workspace access
73
+ finally:
74
+ db.close()
75
+ yield
76
+
77
+
78
+ app = FastAPI(title="ProveKit", version="0.1.0", lifespan=lifespan)
79
+
80
+ # add_middleware inserts at the front, so the LAST one added is the OUTERMOST. Resulting
81
+ # order: CORS -> SecurityHeaders -> RequestID -> BodySizeLimit -> app. Both SecurityHeaders
82
+ # and RequestID must stay outside BodySizeLimit, whose 413 short-circuits the rest of the
83
+ # stack — inside them, that response would carry no security headers and no X-Request-ID.
84
+ app.add_middleware(BodySizeLimitMiddleware)
85
+ app.add_middleware(RequestIDMiddleware)
86
+ app.add_middleware(SecurityHeadersMiddleware)
87
+ app.add_middleware(
88
+ CORSMiddleware,
89
+ allow_origins=settings.cors_origin_list,
90
+ allow_credentials=True, # session cookies
91
+ allow_methods=["*"],
92
+ allow_headers=["*"],
93
+ )
94
+
95
+ app.include_router(auth.router)
96
+ app.include_router(connections.router)
97
+ app.include_router(library.router)
98
+ app.include_router(prompts.router)
99
+ app.include_router(flows.router)
100
+ app.include_router(run.router)
101
+ app.include_router(traces.router)
102
+ app.include_router(traces.ws_router)
103
+ app.include_router(deployments.router)
104
+ app.include_router(runtime.router)
105
+ app.include_router(usage.router)
106
+
107
+
108
+ @app.get("/")
109
+ def root():
110
+ return {"service": "ProveKit", "docs": "/docs"}
111
+
112
+
113
+ @app.get("/healthz")
114
+ def health():
115
+ return healthz()
@@ -0,0 +1,40 @@
1
+ """Alembic environment — pulls the URL and metadata from the app so migrations track models."""
2
+ from logging.config import fileConfig
3
+
4
+ from alembic import context
5
+ from sqlalchemy import create_engine, pool
6
+
7
+ from provekit.config import get_settings
8
+ from provekit.database import Base
9
+ from provekit import models # noqa: F401 (register all tables on Base.metadata)
10
+
11
+ config = context.config
12
+ if config.config_file_name is not None:
13
+ fileConfig(config.config_file_name)
14
+
15
+ # The URL is read straight from settings and handed to create_engine below rather than
16
+ # stored on the alembic config: config values go through ConfigParser interpolation, so a
17
+ # DATABASE_URL whose password contains '%' would raise "invalid interpolation syntax".
18
+ target_metadata = Base.metadata
19
+
20
+
21
+ def run_migrations_offline() -> None:
22
+ context.configure(url=get_settings().database_url, target_metadata=target_metadata,
23
+ literal_binds=True, render_as_batch=True)
24
+ with context.begin_transaction():
25
+ context.run_migrations()
26
+
27
+
28
+ def run_migrations_online() -> None:
29
+ connectable = create_engine(get_settings().database_url, poolclass=pool.NullPool)
30
+ with connectable.connect() as connection:
31
+ # batch mode keeps ALTERs working on SQLite too
32
+ context.configure(connection=connection, target_metadata=target_metadata, render_as_batch=True)
33
+ with context.begin_transaction():
34
+ context.run_migrations()
35
+
36
+
37
+ if context.is_offline_mode():
38
+ run_migrations_offline()
39
+ else:
40
+ run_migrations_online()
@@ -0,0 +1,22 @@
1
+ """${message}
2
+
3
+ Revision ID: ${up_revision}
4
+ Revises: ${down_revision | comma,n}
5
+ Create Date: ${create_date}
6
+ """
7
+ from alembic import op
8
+ import sqlalchemy as sa
9
+ ${imports if imports else ""}
10
+
11
+ revision = ${repr(up_revision)}
12
+ down_revision = ${repr(down_revision)}
13
+ branch_labels = ${repr(branch_labels)}
14
+ depends_on = ${repr(depends_on)}
15
+
16
+
17
+ def upgrade() -> None:
18
+ ${upgrades if upgrades else "pass"}
19
+
20
+
21
+ def downgrade() -> None:
22
+ ${downgrades if downgrades else "pass"}
@@ -0,0 +1,32 @@
1
+ """user email_verified
2
+
3
+ Revision ID: 323eb73d463c
4
+ Revises: 819ed5ff183e
5
+ Create Date: 2026-07-17 05:32:23.391178
6
+ """
7
+ from alembic import op
8
+ import sqlalchemy as sa
9
+
10
+
11
+ revision = '323eb73d463c'
12
+ down_revision = '819ed5ff183e'
13
+ branch_labels = None
14
+ depends_on = None
15
+
16
+
17
+ def upgrade() -> None:
18
+ # ### commands auto generated by Alembic - please adjust! ###
19
+ with op.batch_alter_table('users', schema=None) as batch_op:
20
+ # server_default so existing rows get a value; existing accounts are treated as
21
+ # unverified (they can still sign in unless REQUIRE_EMAIL_VERIFICATION is on).
22
+ batch_op.add_column(sa.Column('email_verified', sa.Boolean(), nullable=False, server_default=sa.false()))
23
+
24
+ # ### end Alembic commands ###
25
+
26
+
27
+ def downgrade() -> None:
28
+ # ### commands auto generated by Alembic - please adjust! ###
29
+ with op.batch_alter_table('users', schema=None) as batch_op:
30
+ batch_op.drop_column('email_verified')
31
+
32
+ # ### end Alembic commands ###