sable-sphynx 0.7.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
  2. sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
  3. sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
  4. sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
  5. sphynx/__init__.py +469 -0
  6. sphynx/__init__.pyi +376 -0
  7. sphynx/_analytical_client.py +77 -0
  8. sphynx/_analytical_values.py +252 -0
  9. sphynx/_audit_client.py +173 -0
  10. sphynx/_comparison_adapters.py +113 -0
  11. sphynx/_componentized_full_rag.py +34 -0
  12. sphynx/_composed_pipeline.py +109 -0
  13. sphynx/_full_rag_grading.py +218 -0
  14. sphynx/_full_rag_trace.py +221 -0
  15. sphynx/_local_eval.py +268 -0
  16. sphynx/_pipeline_core.py +243 -0
  17. sphynx/_pr_client.py +237 -0
  18. sphynx/_rag_diagnostics_calibration.py +285 -0
  19. sphynx/_rag_diagnostics_catalog.py +39 -0
  20. sphynx/_rag_diagnostics_common.py +26 -0
  21. sphynx/_rag_diagnostics_contracts.py +186 -0
  22. sphynx/_rag_diagnostics_engine.py +243 -0
  23. sphynx/_rag_diagnostics_importers.py +242 -0
  24. sphynx/_rag_diagnostics_ranking.py +81 -0
  25. sphynx/_rag_diagnostics_semantic.py +325 -0
  26. sphynx/_rag_ranking.py +149 -0
  27. sphynx/_speech_benchmark_scoring.py +140 -0
  28. sphynx/_speech_benchmark_sources.py +275 -0
  29. sphynx/_speech_complete_scoring.py +218 -0
  30. sphynx/_speech_complete_source.py +118 -0
  31. sphynx/_speech_kido.py +294 -0
  32. sphynx/_speech_scoring.py +77 -0
  33. sphynx/_track_capture.py +213 -0
  34. sphynx/analytical_views.py +515 -0
  35. sphynx/assessment_contracts.py +378 -0
  36. sphynx/assessment_selection.py +57 -0
  37. sphynx/audit_access.py +245 -0
  38. sphynx/audit_contracts.py +204 -0
  39. sphynx/audit_requests.py +180 -0
  40. sphynx/authoring.py +188 -0
  41. sphynx/benchmarks.py +568 -0
  42. sphynx/candidates.py +62 -0
  43. sphynx/catalog.py +272 -0
  44. sphynx/chunking.py +372 -0
  45. sphynx/clickhouse_reader.py +169 -0
  46. sphynx/client.py +486 -0
  47. sphynx/comparison_core.py +149 -0
  48. sphynx/comparison_views.py +223 -0
  49. sphynx/components.py +1661 -0
  50. sphynx/contracts.py +514 -0
  51. sphynx/customer_rerank.py +299 -0
  52. sphynx/customer_retrieval.py +146 -0
  53. sphynx/data/benchmark-releases.v1.json +164 -0
  54. sphynx/data/public-benchmark-release.v1.json +354 -0
  55. sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
  56. sphynx/data/rag-paper-references.v1.json +176 -0
  57. sphynx/data/rag-paper-schemas.v1.json +1069 -0
  58. sphynx/data/rag-rerank-v1.json +507 -0
  59. sphynx/dataset_catalog_data.py +1214 -0
  60. sphynx/dataset_readiness.py +138 -0
  61. sphynx/dataset_v3.py +999 -0
  62. sphynx/demo.py +130 -0
  63. sphynx/demo_candidates/__init__.py +1 -0
  64. sphynx/demo_candidates/dense.py +11 -0
  65. sphynx/demo_candidates/lexical.py +21 -0
  66. sphynx/diagnostics.py +207 -0
  67. sphynx/documentation.py +737 -0
  68. sphynx/documentation_candidate.py +939 -0
  69. sphynx/evals.py +538 -0
  70. sphynx/evaluation_protocols.py +124 -0
  71. sphynx/evaluator_bindings.py +370 -0
  72. sphynx/evidence.py +352 -0
  73. sphynx/execution_capabilities.py +109 -0
  74. sphynx/fast_search.py +285 -0
  75. sphynx/full_rag.py +341 -0
  76. sphynx/hosted/__init__.py +18 -0
  77. sphynx/hosted/benchmark_intake.py +218 -0
  78. sphynx/hosted/contracts.py +361 -0
  79. sphynx/hosted/model_admission.py +130 -0
  80. sphynx/hosted/model_artifacts.py +298 -0
  81. sphynx/hosted/model_contracts.py +63 -0
  82. sphynx/hosted/model_gcs.py +181 -0
  83. sphynx/hosted/model_history.py +84 -0
  84. sphynx/hosted/model_session.py +233 -0
  85. sphynx/hosted/public_suite.py +74 -0
  86. sphynx/hosted/registry.py +97 -0
  87. sphynx/hosted/revisions.py +126 -0
  88. sphynx/hosted/sandbox.py +105 -0
  89. sphynx/hosted/submission.py +287 -0
  90. sphynx/hosted_workflow.py +451 -0
  91. sphynx/http_transport.py +86 -0
  92. sphynx/integrated_rag.py +352 -0
  93. sphynx/integrated_workflow.py +552 -0
  94. sphynx/integration_contracts.py +474 -0
  95. sphynx/invocation.py +791 -0
  96. sphynx/invocation_candidates.py +392 -0
  97. sphynx/invocation_catalog_data.py +40 -0
  98. sphynx/learned_catalog_data.py +30 -0
  99. sphynx/learned_rerank.py +267 -0
  100. sphynx/legacy_full_rag.py +67 -0
  101. sphynx/lifecycle.py +885 -0
  102. sphynx/lifecycle_workflow.py +490 -0
  103. sphynx/metric_views.py +474 -0
  104. sphynx/metrics.py +282 -0
  105. sphynx/model_adapters.py +438 -0
  106. sphynx/model_catalog_data.py +30 -0
  107. sphynx/official_benchmarks.py +385 -0
  108. sphynx/packs.py +258 -0
  109. sphynx/pipeline.py +313 -0
  110. sphynx/pipeline_workflow.py +551 -0
  111. sphynx/pr_evaluations.py +560 -0
  112. sphynx/progress.py +89 -0
  113. sphynx/public_bundle.py +173 -0
  114. sphynx/public_eval.py +62 -0
  115. sphynx/public_rerank.py +355 -0
  116. sphynx/py.typed +0 -0
  117. sphynx/pytest_plugin.py +62 -0
  118. sphynx/question_robustness.py +437 -0
  119. sphynx/rag.py +329 -0
  120. sphynx/rag_diagnostics.py +309 -0
  121. sphynx/rag_papers/__init__.py +29 -0
  122. sphynx/rag_papers/__main__.py +45 -0
  123. sphynx/rag_papers/candidates.py +19 -0
  124. sphynx/rag_papers/contracts.py +282 -0
  125. sphynx/rag_papers/datasets.py +291 -0
  126. sphynx/rag_papers/experiments.py +116 -0
  127. sphynx/rag_papers/judges.py +308 -0
  128. sphynx/rag_papers/registry.py +33 -0
  129. sphynx/rag_papers/runner.py +176 -0
  130. sphynx/rag_papers/sable.py +135 -0
  131. sphynx/rag_papers/schemas.py +12 -0
  132. sphynx/rag_papers/scoring.py +249 -0
  133. sphynx/rag_papers/semantic.py +288 -0
  134. sphynx/rag_papers/trec.py +80 -0
  135. sphynx/recorded_scores.py +352 -0
  136. sphynx/records.py +124 -0
  137. sphynx/reflex_traces.py +193 -0
  138. sphynx/report_rendering.py +71 -0
  139. sphynx/rerank_family_contracts.py +150 -0
  140. sphynx/rerank_workflow.py +424 -0
  141. sphynx/research_candidate.py +190 -0
  142. sphynx/results.py +852 -0
  143. sphynx/retrieval.py +750 -0
  144. sphynx/retrieval_candidates.py +295 -0
  145. sphynx/retrieval_catalog_data.py +19 -0
  146. sphynx/retrieval_recordings.py +125 -0
  147. sphynx/reviewer_audit.py +266 -0
  148. sphynx/routing_policies.py +375 -0
  149. sphynx/runner.py +544 -0
  150. sphynx/sable_rerank.py +489 -0
  151. sphynx/score_reliability.py +393 -0
  152. sphynx/speech_benchmarks.py +204 -0
  153. sphynx/speech_campaigns.py +378 -0
  154. sphynx/speech_complete.py +236 -0
  155. sphynx/speech_contracts.py +225 -0
  156. sphynx/speech_evidence.py +429 -0
  157. sphynx/speech_listening.py +115 -0
  158. sphynx/speech_multilingual.py +71 -0
  159. sphynx/speech_tts.py +479 -0
  160. sphynx/sql/001_sync_v1.sql +30 -0
  161. sphynx/stage_contracts.py +199 -0
  162. sphynx/statistical_catalog_data.py +53 -0
  163. sphynx/statistical_rerank.py +232 -0
  164. sphynx/sync.py +797 -0
  165. sphynx/workbench.py +378 -0
  166. sphynx/workbench_catalog_data.py +13 -0
  167. sphynx/workbench_tracks.py +78 -0
  168. sphynx/workflow_async.py +197 -0
  169. sphynx/workflow_contracts.py +594 -0
  170. sphynx/workflow_decisions.py +211 -0
  171. sphynx/workflow_defaults.py +16 -0
  172. sphynx/workflow_extensions.py +212 -0
  173. sphynx/workflow_registry.py +513 -0
  174. sphynx/workflows.py +792 -0
sphynx/data/rag-paper-references.v1.json ADDED
@@ -0,0 +1,176 @@
1
+ [
2
+ {
3
+ "repository": "beir-cellar/beir",
4
+ "commit": "ef83d29307061c65d04b035b4f4e7c18bd8374af",
5
+ "files": {
6
+ "beir/retrieval/evaluation.py": {
7
+ "sha256": "f346bd20f258e552895e438257e3cc91308b65e0b1ee6c09422e987084bc3bd0",
8
+ "url": "https://raw.githubusercontent.com/beir-cellar/beir/ef83d29307061c65d04b035b4f4e7c18bd8374af/beir/retrieval/evaluation.py"
9
+ }
10
+ }
11
+ },
12
+ {
13
+ "repository": "hotpotqa/hotpot",
14
+ "commit": "3635853403a8735609ee997664e1528f4480762a",
15
+ "files": {
16
+ "hotpot_evaluate_v1.py": {
17
+ "sha256": "d35fc91a6db21d791dbdda11daf3856e9359f5701d54e3eefba20d88fecc02c0",
18
+ "url": "https://raw.githubusercontent.com/hotpotqa/hotpot/3635853403a8735609ee997664e1528f4480762a/hotpot_evaluate_v1.py"
19
+ }
20
+ }
21
+ },
22
+ {
23
+ "repository": "princeton-nlp/ALCE",
24
+ "commit": "246c476a4edfc564266b7346b6e29ef4861ae937",
25
+ "files": {
26
+ "eval.py": {
27
+ "sha256": "e0f63bf865cacc64d7390fc62669c82b2653b1628f40051de0efdf5518064091",
28
+ "url": "https://raw.githubusercontent.com/princeton-nlp/ALCE/246c476a4edfc564266b7346b6e29ef4861ae937/eval.py"
29
+ },
30
+ "utils.py": {
31
+ "sha256": "92e6900b5350f7da4dc179f3d9f498f73d976377a412e8c244db8339ad7976b9",
32
+ "url": "https://raw.githubusercontent.com/princeton-nlp/ALCE/246c476a4edfc564266b7346b6e29ef4861ae937/utils.py"
33
+ }
34
+ }
35
+ },
36
+ {
37
+ "repository": "shmsw25/FActScore",
38
+ "commit": "f28272deffcf33efc1f1117d5479c10bb75221a9",
39
+ "files": {
40
+ "factscore/factscorer.py": {
41
+ "sha256": "d6f61090ad3e2d0cdb84c46c48d27278c2be3c2ae5f3fec2268771a8413e4cdb",
42
+ "url": "https://raw.githubusercontent.com/shmsw25/FActScore/f28272deffcf33efc1f1117d5479c10bb75221a9/factscore/factscorer.py"
43
+ }
44
+ }
45
+ },
46
+ {
47
+ "repository": "amazon-science/RAGChecker",
48
+ "commit": "6091f08c00e676e87a970f2aeb4a23a484746348",
49
+ "files": {
50
+ "ragchecker/metrics.py": {
51
+ "sha256": "e72cfc28a83268df6c32177fb3dac6f9f7fd52363a3e57031d5292a476c05171",
52
+ "url": "https://raw.githubusercontent.com/amazon-science/RAGChecker/6091f08c00e676e87a970f2aeb4a23a484746348/ragchecker/metrics.py"
53
+ },
54
+ "ragchecker/evaluator.py": {
55
+ "url": "https://raw.githubusercontent.com/amazon-science/RAGChecker/6091f08c00e676e87a970f2aeb4a23a484746348/ragchecker/evaluator.py",
56
+ "sha256": "ffe74c4e8551bdeb52a505d85f2edc6571c84ec56c6ed5e2fd7108ae0c52a016"
57
+ },
58
+ "ragchecker/computation.py": {
59
+ "url": "https://raw.githubusercontent.com/amazon-science/RAGChecker/6091f08c00e676e87a970f2aeb4a23a484746348/ragchecker/computation.py",
60
+ "sha256": "18a92c5ef9e5a1d4a1e70d1203cbc281a3061908a247ab6c21411ea4ffa6d574"
61
+ }
62
+ }
63
+ },
64
+ {
65
+ "repository": "StonyBrookNLP/musique",
66
+ "commit": "922ac98f19a201998dbdae6d7f2887a5258dbdeb",
67
+ "files": {
68
+ "evaluate_v1.0.py": {
69
+ "sha256": "f5fe66ae61dbea5172cba9d428d9924a5811f3457edc945fc1d81369c30e74b7",
70
+ "url": "https://raw.githubusercontent.com/StonyBrookNLP/musique/922ac98f19a201998dbdae6d7f2887a5258dbdeb/evaluate_v1.0.py"
71
+ },
72
+ "metrics/answer.py": {
73
+ "sha256": "10368f619b4d5ef5d83748c05a96c0afd332a14ab5c010740c98d58dfaefe974",
74
+ "url": "https://raw.githubusercontent.com/StonyBrookNLP/musique/922ac98f19a201998dbdae6d7f2887a5258dbdeb/metrics/answer.py"
75
+ },
76
+ "metrics/support.py": {
77
+ "sha256": "ac16c0daf458a6a4d6db97682c2340fe5b5936a947bf32c04dc3bf16406077c6",
78
+ "url": "https://raw.githubusercontent.com/StonyBrookNLP/musique/922ac98f19a201998dbdae6d7f2887a5258dbdeb/metrics/support.py"
79
+ }
80
+ }
81
+ },
82
+ {
83
+ "repository": "chen700564/RGB",
84
+ "commit": "65ec39e40e7dc9abb50e9bf1b4f32be3f6f16615",
85
+ "files": {
86
+ "evalue.py": {
87
+ "url": "https://raw.githubusercontent.com/chen700564/RGB/65ec39e40e7dc9abb50e9bf1b4f32be3f6f16615/evalue.py",
88
+ "sha256": "506436ffbd7b97a6cadeb34961f4b4a4a9f04bf8afe7a1729e467811243352c0"
89
+ },
90
+ "fact_evalue.py": {
91
+ "url": "https://raw.githubusercontent.com/chen700564/RGB/65ec39e40e7dc9abb50e9bf1b4f32be3f6f16615/fact_evalue.py",
92
+ "sha256": "c903ac0cc4f795fc9c45569ce62340f762420fc7f2a6ab7f646a7876577b803d"
93
+ },
94
+ "reject_evalue.py": {
95
+ "url": "https://raw.githubusercontent.com/chen700564/RGB/65ec39e40e7dc9abb50e9bf1b4f32be3f6f16615/reject_evalue.py",
96
+ "sha256": "06771cdcd778c535c1d29cafde1246d399780ce64b33914567dfcbcc26dd3d4e"
97
+ },
98
+ "readme.md": {
99
+ "url": "https://raw.githubusercontent.com/chen700564/RGB/65ec39e40e7dc9abb50e9bf1b4f32be3f6f16615/readme.md",
100
+ "sha256": "e924a72f4b46718dde52cd3eb603a7f6496a9658e931e3269461a88bf6b86c8a"
101
+ }
102
+ }
103
+ },
104
+ {
105
+ "repository": "facebookresearch/CRAG",
106
+ "commit": "ad1518887dd4d9ebcd7de95388c7a62751e7705c",
107
+ "files": {
108
+ "local_evaluation.py": {
109
+ "url": "https://raw.githubusercontent.com/facebookresearch/CRAG/ad1518887dd4d9ebcd7de95388c7a62751e7705c/local_evaluation.py",
110
+ "sha256": "c6ade99efc3f7fcc8d8bc44970a6d41346358b930cfaaf03945912c23bb4091e"
111
+ },
112
+ "docs/dataset.md": {
113
+ "url": "https://raw.githubusercontent.com/facebookresearch/CRAG/ad1518887dd4d9ebcd7de95388c7a62751e7705c/docs/dataset.md",
114
+ "sha256": "4f920d3c770d630743d23b38310525387bbc1e256876b33d0e4a7f324c7b9269"
115
+ }
116
+ }
117
+ },
118
+ {
119
+ "repository": "allenai/qasper-led-baseline",
120
+ "commit": "afd0fb96bf78ce8cd8157639c6f6a6995e4f9089",
121
+ "files": {
122
+ "scripts/evaluator.py": {
123
+ "sha256": "781aba7cd8e524bef4f0a1b4bf3504e5b02cb1d8d5bf32a8f0a89dfa83e86bfe",
124
+ "url": "https://raw.githubusercontent.com/allenai/qasper-led-baseline/afd0fb96bf78ce8cd8157639c6f6a6995e4f9089/scripts/evaluator.py"
125
+ }
126
+ }
127
+ },
128
+ {
129
+ "repository": "nelson-liu/lost-in-the-middle",
130
+ "commit": "29b8a6d042ce29abccee3db1a73171a107d7e6af",
131
+ "files": {
132
+ "README.md": {
133
+ "sha256": "152bea0f333ea8c3c9872e420a7b5bcf9c51d0e2920cffe01f04259cd952cb25",
134
+ "url": "https://raw.githubusercontent.com/nelson-liu/lost-in-the-middle/29b8a6d042ce29abccee3db1a73171a107d7e6af/README.md"
135
+ }
136
+ }
137
+ },
138
+ {
139
+ "repository": "stanford-futuredata/ARES",
140
+ "commit": "c7c9018a755faf8347c4da415632bae1593ef104",
141
+ "files": {
142
+ "README.md": {
143
+ "sha256": "9c376e573bcfe7ac44c9cb3eb463c917aadefb35785ad861c1a7925c4a3fc6ae",
144
+ "url": "https://raw.githubusercontent.com/stanford-futuredata/ARES/c7c9018a755faf8347c4da415632bae1593ef104/README.md"
145
+ }
146
+ }
147
+ },
148
+ {
149
+ "repository": "explodinggradients/ragas",
150
+ "commit": "298b68274234c060deacab3cf5fb52aa3a20e885",
151
+ "files": {
152
+ "src/ragas/metrics/_faithfulness.py": {
153
+ "sha256": "5370b57f3df537a1619f7c6ddeeb5fd67e392821a5dee30eb5e0830f3a75cdc9",
154
+ "url": "https://raw.githubusercontent.com/explodinggradients/ragas/298b68274234c060deacab3cf5fb52aa3a20e885/src/ragas/metrics/_faithfulness.py"
155
+ },
156
+ "src/ragas/metrics/_context_precision.py": {
157
+ "sha256": "baaf3981986602754ccbaef2936b6ac8e6d6551cab86477e64170683d74a4716",
158
+ "url": "https://raw.githubusercontent.com/explodinggradients/ragas/298b68274234c060deacab3cf5fb52aa3a20e885/src/ragas/metrics/_context_precision.py"
159
+ },
160
+ "src/ragas/metrics/_answer_relevance.py": {
161
+ "sha256": "f7d515ec78522ab48d5aa3212e875e15e1fb3163bae7c0f31637262fd3ef3c25",
162
+ "url": "https://raw.githubusercontent.com/explodinggradients/ragas/298b68274234c060deacab3cf5fb52aa3a20e885/src/ragas/metrics/_answer_relevance.py"
163
+ }
164
+ }
165
+ },
166
+ {
167
+ "repository": "hljoren/sufficientcontext",
168
+ "commit": "801b465eb3be5bf5216fbe3301e61daee85ff99c",
169
+ "files": {
170
+ "README.md": {
171
+ "url": "https://raw.githubusercontent.com/hljoren/sufficientcontext/801b465eb3be5bf5216fbe3301e61daee85ff99c/README.md",
172
+ "sha256": "16d34ae4d479b4aff9ea50d83eea46192f1b82e370c2338e1b5840261548e0ff"
173
+ }
174
+ }
175
+ }
176
+ ]