ragas 0.2.3__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (312) hide show
  1. {ragas-0.2.3 → ragas-0.2.4}/.readthedocs.yml +0 -1
  2. {ragas-0.2.3 → ragas-0.2.4}/Makefile +1 -4
  3. {ragas-0.2.3/src/ragas.egg-info → ragas-0.2.4}/PKG-INFO +3 -2
  4. {ragas-0.2.3 → ragas-0.2.4}/docs/community/index.md +11 -8
  5. ragas-0.2.4/docs/extra/components/choose_evaluator_llm.md +140 -0
  6. ragas-0.2.4/docs/extra/components/choose_generator_llm.md +138 -0
  7. {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/rag_testset_generation.md +4 -4
  8. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/customize_models.md +1 -1
  9. ragas-0.2.4/docs/howtos/customizations/metrics/_write_your_own_metric.md +298 -0
  10. ragas-0.2.3/docs/howtos/customizations/metrics/_write_your_own_metric.md → ragas-0.2.4/docs/howtos/customizations/metrics/_write_your_own_metric_advanced.md +4 -6
  11. ragas-0.2.4/docs/howtos/customizations/metrics/write_your_own_metric.ipynb +470 -0
  12. ragas-0.2.3/docs/howtos/customizations/metrics/write_your_own_metric.ipynb → ragas-0.2.4/docs/howtos/customizations/metrics/write_your_own_metric_advanced.ipynb +4 -6
  13. ragas-0.2.4/docs/howtos/integrations/_llamaindex.md +371 -0
  14. ragas-0.2.4/docs/howtos/integrations/llamaindex.ipynb +653 -0
  15. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/migrations/migrate_from_v01_to_v02.md +1 -1
  16. {ragas-0.2.3/scripts → ragas-0.2.4/docs}/ipynb_to_md.py +18 -16
  17. {ragas-0.2.3 → ragas-0.2.4}/docs/references/testset_schema.md +2 -2
  18. {ragas-0.2.3 → ragas-0.2.4}/mkdocs.yml +10 -4
  19. {ragas-0.2.3 → ragas-0.2.4}/pyproject.toml +2 -1
  20. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/_version.py +2 -2
  21. ragas-0.2.4/src/ragas/async_utils.py +89 -0
  22. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/callbacks.py +13 -12
  23. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/dataset_schema.py +53 -9
  24. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/evaluation.py +18 -6
  25. ragas-0.2.4/src/ragas/executor.py +226 -0
  26. ragas-0.2.4/src/ragas/integrations/llama_index.py +104 -0
  27. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/llms/base.py +13 -5
  28. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_answer_correctness.py +3 -3
  29. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_answer_relevance.py +1 -1
  30. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_answer_similarity.py +9 -5
  31. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_aspect_critic.py +10 -5
  32. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_bleu_score.py +1 -1
  33. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_context_entities_recall.py +1 -1
  34. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_context_precision.py +5 -5
  35. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_context_recall.py +2 -2
  36. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_datacompy_score.py +1 -1
  37. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_domain_specific_rubrics.py +29 -16
  38. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_factual_correctness.py +1 -3
  39. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_faithfulness.py +2 -2
  40. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_goal_accuracy.py +2 -2
  41. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_instance_specific_rubrics.py +2 -2
  42. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_multi_modal_faithfulness.py +1 -1
  43. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_multi_modal_relevance.py +1 -1
  44. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_noise_sensitivity.py +3 -4
  45. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_rouge_score.py +1 -1
  46. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_simple_criteria.py +7 -7
  47. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_sql_semantic_equivalence.py +1 -1
  48. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_string.py +5 -3
  49. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_summarization.py +1 -1
  50. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_tool_call_accuracy.py +4 -2
  51. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_topic_adherence.py +12 -16
  52. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/base.py +40 -9
  53. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/__init__.py +5 -1
  54. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/graph.py +84 -16
  55. ragas-0.2.4/src/ragas/testset/graph_queries.py +38 -0
  56. ragas-0.2.4/src/ragas/testset/persona.py +146 -0
  57. ragas-0.2.4/src/ragas/testset/synthesizers/__init__.py +29 -0
  58. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/base.py +11 -2
  59. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/generate.py +166 -9
  60. ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/__init__.py +10 -0
  61. ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/abstract.py +118 -0
  62. ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/base.py +181 -0
  63. ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/prompts.py +86 -0
  64. ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/specific.py +116 -0
  65. ragas-0.2.4/src/ragas/testset/synthesizers/prompts.py +49 -0
  66. ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/__init__.py +3 -0
  67. ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/base.py +145 -0
  68. ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/prompts.py +35 -0
  69. ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/specific.py +90 -0
  70. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/base.py +13 -15
  71. ragas-0.2.4/src/ragas/testset/transforms/default.py +93 -0
  72. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/engine.py +3 -1
  73. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/__init__.py +2 -0
  74. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/llm_based.py +135 -45
  75. ragas-0.2.4/src/ragas/testset/transforms/relationship_builders/__init__.py +4 -0
  76. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/relationship_builders/cosine.py +0 -46
  77. ragas-0.2.4/src/ragas/testset/transforms/relationship_builders/traditional.py +155 -0
  78. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/splitters/headline.py +24 -2
  79. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/utils.py +20 -0
  80. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/validation.py +3 -2
  81. {ragas-0.2.3 → ragas-0.2.4/src/ragas.egg-info}/PKG-INFO +3 -2
  82. {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/SOURCES.txt +16 -4
  83. {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/requires.txt +2 -1
  84. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/prompt/test_prompt_mixin.py +3 -3
  85. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_analytics.py +5 -5
  86. ragas-0.2.4/tests/unit/test_async_utils.py +38 -0
  87. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_executor.py +43 -22
  88. ragas-0.2.4/tests/unit/test_executor_in_jupyter.ipynb +307 -0
  89. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_metric.py +33 -1
  90. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_prompt.py +8 -8
  91. ragas-0.2.4/tests/unit/test_utils.py +89 -0
  92. ragas-0.2.3/docs/extra/components/choose_evaluator_llm.md +0 -68
  93. ragas-0.2.3/docs/extra/components/choose_generator_llm.md +0 -67
  94. ragas-0.2.3/docs/howtos/integrations/_llamaindex.md +0 -407
  95. ragas-0.2.3/docs/howtos/integrations/llamaindex.ipynb +0 -723
  96. ragas-0.2.3/src/ragas/async_utils.py +0 -49
  97. ragas-0.2.3/src/ragas/executor.py +0 -167
  98. ragas-0.2.3/src/ragas/integrations/llama_index.py +0 -101
  99. ragas-0.2.3/src/ragas/testset/synthesizers/__init__.py +0 -38
  100. ragas-0.2.3/src/ragas/testset/synthesizers/abstract_query.py +0 -349
  101. ragas-0.2.3/src/ragas/testset/synthesizers/base_query.py +0 -95
  102. ragas-0.2.3/src/ragas/testset/synthesizers/prompts.py +0 -408
  103. ragas-0.2.3/src/ragas/testset/synthesizers/specific_query.py +0 -115
  104. ragas-0.2.3/src/ragas/testset/transforms/default.py +0 -80
  105. ragas-0.2.3/src/ragas/testset/transforms/relationship_builders/__init__.py +0 -3
  106. ragas-0.2.3/tests/unit/test_executor_in_jupyter.ipynb +0 -190
  107. ragas-0.2.3/tests/unit/test_utils.py +0 -53
  108. {ragas-0.2.3 → ragas-0.2.4}/.dockerignore +0 -0
  109. {ragas-0.2.3 → ragas-0.2.4}/.gitattributes +0 -0
  110. {ragas-0.2.3 → ragas-0.2.4}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  111. {ragas-0.2.3 → ragas-0.2.4}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  112. {ragas-0.2.3 → ragas-0.2.4}/.github/ISSUE_TEMPLATE/question.md +0 -0
  113. {ragas-0.2.3 → ragas-0.2.4}/.github/workflows/ci.yaml +0 -0
  114. {ragas-0.2.3 → ragas-0.2.4}/.github/workflows/issue-manager.yaml +0 -0
  115. {ragas-0.2.3 → ragas-0.2.4}/.github/workflows/python-publish.yml +0 -0
  116. {ragas-0.2.3 → ragas-0.2.4}/.gitignore +0 -0
  117. {ragas-0.2.3 → ragas-0.2.4}/DEVELOPMENT.md +0 -0
  118. {ragas-0.2.3 → ragas-0.2.4}/LICENSE +0 -0
  119. {ragas-0.2.3 → ragas-0.2.4}/README.md +0 -0
  120. {ragas-0.2.3 → ragas-0.2.4}/docs/Makefile +0 -0
  121. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_ipython3.css +0 -0
  122. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_ipython3_dark.css +0 -0
  123. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_ipython3_light.css +0 -0
  124. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_python.css +0 -0
  125. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_python_dark.css +0 -0
  126. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_python_light.css +0 -0
  127. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/ragas.css +0 -0
  128. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/favicon.ico +0 -0
  129. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/bar-graph.svg +0 -0
  130. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-emb-results.png +0 -0
  131. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-embeddings.jpeg +0 -0
  132. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-llm-result.png +0 -0
  133. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-llms-front.jpeg +0 -0
  134. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-llms-testset.png +0 -0
  135. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/component-wise-metrics.png +0 -0
  136. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/eval-evolve.png +0 -0
  137. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/kg_rag.png +0 -0
  138. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/langsmith-tracing-faithfullness.png +0 -0
  139. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/langsmith-tracing-overview.png +0 -0
  140. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/logo-black.svg +0 -0
  141. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/logo.png +0 -0
  142. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/metrics_mindmap.png +0 -0
  143. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/question_types.png +0 -0
  144. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/quickstart-output.png +0 -0
  145. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/ragas-logo.png +0 -0
  146. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/ragas_workflow_white.png +0 -0
  147. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/scenario_rag.png +0 -0
  148. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/testset_output.png +0 -0
  149. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/trace-langsmith.png +0 -0
  150. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/header_border.js +0 -0
  151. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/mathjax.js +0 -0
  152. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/mendable_chat_bubble.js +0 -0
  153. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/octolane.js +0 -0
  154. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/toggle.js +0 -0
  155. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-dataset.png +0 -0
  156. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-evaluation.png +0 -0
  157. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-feedback.png +0 -0
  158. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-ragas-chain-trace.png +0 -0
  159. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/list-score-traces-ragas.png +0 -0
  160. {ragas-0.2.3 → ragas-0.2.4}/docs/_static/traces-score-ragas.png +0 -0
  161. {ragas-0.2.3 → ragas-0.2.4}/docs/alfred.py +0 -0
  162. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/eval_dataset.md +0 -0
  163. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/eval_sample.md +0 -0
  164. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/index.md +0 -0
  165. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/prompt.md +0 -0
  166. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/feedback/index.md +0 -0
  167. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/index.md +0 -0
  168. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/agents.md +0 -0
  169. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/answer_correctness.md +0 -0
  170. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/answer_relevance.md +0 -0
  171. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/aspect_critic.md +0 -0
  172. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/context_entities_recall.md +0 -0
  173. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/context_precision.md +0 -0
  174. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/context_recall.md +0 -0
  175. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/factual_correctness.md +0 -0
  176. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/faithfulness.md +0 -0
  177. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/general_purpose.md +0 -0
  178. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/index.md +0 -0
  179. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/multi_modal_faithfulness.md +0 -0
  180. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/multi_modal_relevance.md +0 -0
  181. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/noise_sensitivity.md +0 -0
  182. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/rubrics_based.md +0 -0
  183. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/semantic_similarity.md +0 -0
  184. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/sql.md +0 -0
  185. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/summarization_score.md +0 -0
  186. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/traditional.md +0 -0
  187. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/index.md +0 -0
  188. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/overview/index.md +0 -0
  189. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/test_data_generation/agents.md +0 -0
  190. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/test_data_generation/index.md +0 -0
  191. {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/test_data_generation/rag.md +0 -0
  192. {ragas-0.2.3 → ragas-0.2.4}/docs/conf.py +0 -0
  193. {ragas-0.2.3 → ragas-0.2.4}/docs/extra/fonts/Satoshi-Variable.ttf +0 -0
  194. {ragas-0.2.3 → ragas-0.2.4}/docs/extra/fonts/Satoshi-VariableItalic.ttf +0 -0
  195. {ragas-0.2.3 → ragas-0.2.4}/docs/extra/style.css +0 -0
  196. {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/index.md +0 -0
  197. {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/install.md +0 -0
  198. {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/rag_evaluation.md +0 -0
  199. {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/raga_evaluation_output.png +0 -0
  200. {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/testset_output.png +0 -0
  201. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/_cost.md +0 -0
  202. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/add_to_ci.md +0 -0
  203. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/compare_embeddings.md +0 -0
  204. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/compare_llms.md +0 -0
  205. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/cost.ipynb +0 -0
  206. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/index.md +0 -0
  207. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/_run_config.md +0 -0
  208. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/index.md +0 -0
  209. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/_metrics_language_adaptation.md +0 -0
  210. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/_modifying-prompts-metrics.md +0 -0
  211. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/metrics_language_adaptation.ipynb +0 -0
  212. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/modifying-prompts-metrics.ipynb +0 -0
  213. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/tracing.md +0 -0
  214. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/run_config.ipynb +0 -0
  215. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/testgenerator/index.md +0 -0
  216. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/index.md +0 -0
  217. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_arize.md +0 -0
  218. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_athina.md +0 -0
  219. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_haystack.md +0 -0
  220. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_helicone.md +0 -0
  221. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_langchain.md +0 -0
  222. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_langfuse.md +0 -0
  223. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_langsmith.md +0 -0
  224. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_openlayer.md +0 -0
  225. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_opik.md +0 -0
  226. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_tonic-validate.md +0 -0
  227. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_zeno.md +0 -0
  228. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/arize.ipynb +0 -0
  229. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/athina.ipynb +0 -0
  230. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/haystack.ipynb +0 -0
  231. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/helicone.ipynb +0 -0
  232. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/index.md +0 -0
  233. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/langchain.ipynb +0 -0
  234. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/langfuse.ipynb +0 -0
  235. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/langsmith.ipynb +0 -0
  236. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/nyc_wikipedia/nyc_text.txt +0 -0
  237. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/openlayer.ipynb +0 -0
  238. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/opik.ipynb +0 -0
  239. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/tonic-validate.ipynb +0 -0
  240. {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/zeno.ipynb +0 -0
  241. {ragas-0.2.3 → ragas-0.2.4}/docs/index.md +0 -0
  242. {ragas-0.2.3 → ragas-0.2.4}/docs/make.bat +0 -0
  243. {ragas-0.2.3 → ragas-0.2.4}/docs/references/embeddings.md +0 -0
  244. {ragas-0.2.3 → ragas-0.2.4}/docs/references/evaluate.md +0 -0
  245. {ragas-0.2.3 → ragas-0.2.4}/docs/references/evaluation_schema.md +0 -0
  246. {ragas-0.2.3 → ragas-0.2.4}/docs/references/executor.md +0 -0
  247. {ragas-0.2.3 → ragas-0.2.4}/docs/references/generate.md +0 -0
  248. {ragas-0.2.3 → ragas-0.2.4}/docs/references/graph.md +0 -0
  249. {ragas-0.2.3 → ragas-0.2.4}/docs/references/integrations.md +0 -0
  250. {ragas-0.2.3 → ragas-0.2.4}/docs/references/llms.md +0 -0
  251. {ragas-0.2.3 → ragas-0.2.4}/docs/references/metrics.md +0 -0
  252. {ragas-0.2.3 → ragas-0.2.4}/docs/references/prompt.md +0 -0
  253. {ragas-0.2.3 → ragas-0.2.4}/docs/references/run_config.md +0 -0
  254. {ragas-0.2.3 → ragas-0.2.4}/docs/references/synthesizers.md +0 -0
  255. {ragas-0.2.3 → ragas-0.2.4}/docs/references/transforms.md +0 -0
  256. {ragas-0.2.3 → ragas-0.2.4}/references.md +0 -0
  257. {ragas-0.2.3 → ragas-0.2.4}/requirements/dev.txt +0 -0
  258. {ragas-0.2.3 → ragas-0.2.4}/requirements/docs.txt +0 -0
  259. {ragas-0.2.3 → ragas-0.2.4}/requirements/test.txt +0 -0
  260. {ragas-0.2.3 → ragas-0.2.4}/setup.cfg +0 -0
  261. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/__init__.py +0 -0
  262. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/_analytics.py +0 -0
  263. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/cost.py +0 -0
  264. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/embeddings/__init__.py +0 -0
  265. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/embeddings/base.py +0 -0
  266. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/exceptions.py +0 -0
  267. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/experimental/__init__.py +0 -0
  268. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/__init__.py +0 -0
  269. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/helicone.py +0 -0
  270. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/langchain.py +0 -0
  271. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/langsmith.py +0 -0
  272. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/opik.py +0 -0
  273. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/llms/__init__.py +0 -0
  274. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/messages.py +0 -0
  275. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/__init__.py +0 -0
  276. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/utils.py +0 -0
  277. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/__init__.py +0 -0
  278. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/base.py +0 -0
  279. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/mixin.py +0 -0
  280. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/multi_modal_prompt.py +0 -0
  281. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/pydantic_prompt.py +0 -0
  282. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/utils.py +0 -0
  283. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/run_config.py +0 -0
  284. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/testset_schema.py +0 -0
  285. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/utils.py +0 -0
  286. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/__init__.py +0 -0
  287. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/embeddings.py +0 -0
  288. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/regex_based.py +0 -0
  289. {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/splitters/__init__.py +0 -0
  290. {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/dependency_links.txt +0 -0
  291. {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/top_level.txt +0 -0
  292. {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/Dockerfile +0 -0
  293. {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/benchmark_eval.py +0 -0
  294. {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/benchmark_testsetgen.py +0 -0
  295. {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/utils.py +0 -0
  296. {ragas-0.2.3 → ragas-0.2.4}/tests/conftest.py +0 -0
  297. {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_adaptation.py +0 -0
  298. {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_amnesty_in_ci.py +0 -0
  299. {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_evaluation_in_jupyter.ipynb +0 -0
  300. {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_fullflow.py +0 -0
  301. {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_langsmith.py +0 -0
  302. {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_testset_gen_in_jupyter.ipynb +0 -0
  303. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/llms/test_llm.py +0 -0
  304. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/prompt/test_prompt_utils.py +0 -0
  305. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_cost.py +0 -0
  306. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_dataset_schema.py +0 -0
  307. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_embeddings.py +0 -0
  308. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_import.py +0 -0
  309. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_run_config.py +0 -0
  310. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_simple.py +0 -0
  311. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_testset_schema.py +0 -0
  312. {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_validation.py +0 -0
@@ -7,5 +7,4 @@ build:
7
7
  commands:
8
8
  - pip install -e .[docs]
9
9
  - if [ -n "$GH_TOKEN" ]; then pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git; fi
10
- - make ipynb-to-md
11
10
  - mkdocs build --site-dir $READTHEDOCS_OUTPUT/html
@@ -39,11 +39,8 @@ run-ci: format lint type test ## Running all CI checks
39
39
  rewrite-docs: ## Use GPT4 to rewrite the documentation
40
40
  @echo "Rewriting the documentation in directory $(DIR)..."
41
41
  $(Q)python $(GIT_ROOT)/docs/python alphred.py --directory $(DIR)
42
- ipynb-to-md: ## Convert ipynb files to md files
43
- $(Q)python $(GIT_ROOT)/scripts/ipynb_to_md.py
44
42
  docsite: ## Build and serve documentation
45
- $(Q)$(MAKE) ipynb-to-md
46
- $(Q)mkdocs serve
43
+ $(Q)mkdocs serve --dirtyreload
47
44
 
48
45
  # Benchmarks
49
46
  run-benchmarks-eval: ## Run benchmarks for Evaluation
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ragas
3
- Version: 0.2.3
3
+ Version: 0.2.4
4
4
  Description-Content-Type: text/markdown
5
5
  License-File: LICENSE
6
6
  Requires-Dist: numpy
@@ -23,10 +23,11 @@ Requires-Dist: rouge_score; extra == "all"
23
23
  Requires-Dist: rapidfuzz; extra == "all"
24
24
  Requires-Dist: pandas; extra == "all"
25
25
  Requires-Dist: datacompy; extra == "all"
26
+ Requires-Dist: llama_index; extra == "all"
26
27
  Provides-Extra: docs
27
28
  Requires-Dist: mkdocs>=1.6.1; extra == "docs"
28
29
  Requires-Dist: mkdocs-material; extra == "docs"
29
- Requires-Dist: mkdocs-jupyter; extra == "docs"
30
+ Requires-Dist: mkdocs-material[imaging]; extra == "docs"
30
31
  Requires-Dist: mkdocstrings[python]; extra == "docs"
31
32
  Requires-Dist: mkdocs-glightbox; extra == "docs"
32
33
  Requires-Dist: mkdocs-autorefs; extra == "docs"
@@ -17,8 +17,8 @@ Meet some of our outstanding members who made significant contributions !
17
17
  Explore insightful articles, tutorials, and stories written by and for our community members.
18
18
 
19
19
  - [Luka Panić](https://www.linkedin.com/in/luka-pani%C4%87-20b671277/) shares his work on
20
- - [Ragas Evaluation: In-Depth Insights | PIXION Blog](https://pixion.co/blog/ragas-evaluation-in-depth-insights): A detailed explanation of the metrics and how they are calculated.
21
- - [RAG in practice - Test Set Generation | PIXION Blog](https://pixion.co/blog/rag-in-practice-test-set-generation): A tutorial on how to generate a test set using Ragas.
20
+ - [Ragas Evaluation: In-Depth Insights | PIXION Blog](https://pixion.co/blog/ragas-evaluation-in-depth-insights): A detailed explanation of the metrics and how they are calculated.
21
+ - [RAG in practice - Test Set Generation | PIXION Blog](https://pixion.co/blog/rag-in-practice-test-set-generation): A tutorial on how to generate a test set using Ragas.
22
22
  - [Shanthi Vardhan](https://www.linkedin.com/in/shanthivardhan/) shares how his team at [Atomicwork uses ragas](https://www.atomicwork.com/blog/ragas-improving-atom-accuracy) to improve their AI system's ability to accurately identify and retrieve more precise information for enhanced service management.
23
23
  - [Pinecone's](https://pinecone.io/blog) study on how RAGs can enhance capabilities of LLMs in ["RAG makes LLMs better and equal"](https://www.pinecone.io/blog/rag-study/) uses ragas to proves context retrieval makes LLMs provide significantly better results, even when increasing the data size to 1 billion.
24
24
  - [Aishwarya Prabhat](https://www.linkedin.com/in/aishwaryaprabhat/) shares her expertise on advanced RAG techniques in her comprehensive guide, ["Performing, Evaluating & Tracking Advanced RAG (ft. AzureML, LlamaIndex & Ragas)"](https://www.linkedin.com/pulse/performing-evaluating-tracking-advanced-rag-ft-azureml-prabhat-i1rkc/).
@@ -34,13 +34,16 @@ Explore insightful articles, tutorials, and stories written by and for our commu
34
34
  - [RAG Evaluation : Computational Metrics in RAG and Calculation Methods in Ragas](https://tech.beatrust.com/entry/2024/05/02/RAG_Evaluation_%3A_Computational_Metrics_in_RAG_and_Calculation_Methods_in_Ragas)
35
35
  - [RAG Evaluation: Assessing the Usefulness of Ragas](https://tech.beatrust.com/entry/2024/05/02/RAG_Evaluation%3A_Assessing_the_Usefulness_of_Ragas)
36
36
  - [Atita Arora](https://www.linkedin.com/in/atitaarora/) writes about [Evaluating Retrieval Augmented Generation using RAGAS](https://superlinked.com/vectorhub/articles/retrieval-augmented-generation-eval-qdrant-ragas), an end-to-end tutorial on building RAG using [Qdrant](https://qdrant.tech/) and [Langchain](https://www.langchain.com/) and evaluating it with RAGAS.
37
- - *Bonus content* : Learn how to create an evaluation dataset that serves as a reference point for evaluating our RAG pipeline, Understand the RAGAS evaluation metrics and how to make sense of them and putting them in action to test a Naive RAG pipeline and measure its performance using RAGAS metrics.
38
- - *Code walkthrough* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas
39
- - *Code walkthrough using [Deepset Haystack](https://haystack.deepset.ai/) and [Mixedbread.ai](https://www.mixedbread.ai/)* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas-haystack
37
+ - *Bonus content* : Learn how to create an evaluation dataset that serves as a reference point for evaluating our RAG pipeline, Understand the RAGAS evaluation metrics and how to make sense of them and putting them in action to test a Naive RAG pipeline and measure its performance using RAGAS metrics.
38
+ - *Code walkthrough* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas
39
+ - *Code walkthrough using [Deepset Haystack](https://haystack.deepset.ai/) and [Mixedbread.ai](https://www.mixedbread.ai/)* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas-haystack
40
40
  - [Minoru Onda](https://x.com/minorun365) writes for beginners about how to start Ragas v0.2 evaluation with Amazon Bedrock, and integrate with Langfuse.
41
- - [RAG精度評価の定番ツール「Ragas」にAWSのBedrockで入門しよう!(v0.2対応) - Qiita](https://qiita.com/minorun365/items/2f4e238f8bbc6e393ba5)
42
- - [生成AIアプリの出力をRagasで評価して、LangfuseでGUI監視しよう! - Qiita](https://qiita.com/minorun365/items/70ad2f5a0afaac6e5cb9)
43
-
41
+ - [RAG精度評価の定番ツール「Ragas」にAWSのBedrockで入門しよう!(v0.2対応) - Qiita](https://qiita.com/minorun365/items/2f4e238f8bbc6e393ba5)
42
+ - [生成AIアプリの出力をRagasで評価して、LangfuseでGUI監視しよう! - Qiita](https://qiita.com/minorun365/items/70ad2f5a0afaac6e5cb9)
43
+ - [Yunnglin](https://github.com/Yunnglin) has penned a guide on integrating Ragas v0.2 into [EvalScope](https://github.com/modelscope/eval-scope) (an evaluation framework for large models), thereby utilizing the [ModelScope](https://github.com/modelscope/modelscope) ecosystem.
44
+ - Tutorial: [Using Ragas with EvalScope](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html)
45
+ - 教程: [在EvalScope中使用Ragas一键发起RAG评估](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/backend/rageval_backend/ragas.html)
46
+ - 最佳实践: [打破文本边界:如何进行多模态RAG评估](https://evalscope.readthedocs.io/zh-cn/latest/blog/RAG/multimodal_RAG.html#multimodal-rag)
44
47
 
45
48
  ## **📅 Events**
46
49
 
@@ -0,0 +1,140 @@
1
+
2
+ === "OpenAI"
3
+ Install the langchain-openai package
4
+
5
+ ```bash
6
+ pip install langchain-openai
7
+ ```
8
+
9
+ Ensure you have your OpenAI key ready and available in your environment.
10
+
11
+ ```python
12
+ import os
13
+ os.environ["OPENAI_API_KEY"] = "your-openai-key"
14
+ ```
15
+ Wrap the LLMs in `LangchainLLMWrapper` so that it can be used with ragas.
16
+
17
+ ```python
18
+ from ragas.llms import LangchainLLMWrapper
19
+ from ragas.embeddings import LangchainEmbeddingsWrapper
20
+ from langchain_openai import ChatOpenAI
21
+ from langchain_openai import OpenAIEmbeddings
22
+ evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
23
+ evaluator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
24
+ ```
25
+
26
+
27
+ === "Amazon Bedrock"
28
+ Install the langchain-aws package
29
+
30
+ ```bash
31
+ pip install langchain-aws
32
+ ```
33
+
34
+ Then you have to set your AWS credentials and configurations
35
+
36
+ ```python
37
+ config = {
38
+ "credentials_profile_name": "your-profile-name", # E.g "default"
39
+ "region_name": "your-region-name", # E.g. "us-east-1"
40
+ "llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
41
+ "embeddings": "your-embedding-model-id", # E.g "amazon.titan-embed-text-v2:0"
42
+ "temperature": 0.4,
43
+ }
44
+ ```
45
+
46
+ Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
47
+
48
+ ```python
49
+ from langchain_aws import ChatBedrockConverse
50
+ from langchain_aws import BedrockEmbeddings
51
+ from ragas.llms import LangchainLLMWrapper
52
+ from ragas.embeddings import LangchainEmbeddingsWrapper
53
+
54
+ evaluator_llm = LangchainLLMWrapper(ChatBedrockConverse(
55
+ credentials_profile_name=config["credentials_profile_name"],
56
+ region_name=config["region_name"],
57
+ base_url=f"https://bedrock-runtime.{config['region_name']}.amazonaws.com",
58
+ model=config["llm"],
59
+ temperature=config["temperature"],
60
+ ))
61
+ evaluator_embeddings = LangchainEmbeddingsWrapper(BedrockEmbeddings(
62
+ credentials_profile_name=config["credentials_profile_name"],
63
+ region_name=config["region_name"],
64
+ model_id=config["embeddings"],
65
+ ))
66
+ ```
67
+
68
+ If you want more information on how to use other AWS services, please refer to the [langchain-aws](https://python.langchain.com/docs/integrations/providers/aws/) documentation.
69
+
70
+ === "Azure OpenAI"
71
+ Install the langchain-openai package
72
+
73
+ ```bash
74
+ pip install langchain-openai
75
+ ```
76
+
77
+ Ensure you have your Azure OpenAI key ready and available in your environment.
78
+
79
+ ```python
80
+ import os
81
+ os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-openai-key"
82
+
83
+ # other configuration
84
+ azure_config = {
85
+ "base_url": "", # your endpoint
86
+ "model_deployment": "", # your model deployment name
87
+ "model_name": "", # your model name
88
+ "embedding_deployment": "", # your embedding deployment name
89
+ "embedding_name": "", # your embedding name
90
+ }
91
+
92
+ ```
93
+
94
+ Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
95
+
96
+ ```python
97
+ from langchain_openai import AzureChatOpenAI
98
+ from langchain_openai import AzureOpenAIEmbeddings
99
+ from ragas.llms import LangchainLLMWrapper
100
+ from ragas.embeddings import LangchainEmbeddingsWrapper
101
+ evaluator_llm = LangchainLLMWrapper(AzureChatOpenAI(
102
+ openai_api_version="2023-05-15",
103
+ azure_endpoint=azure_configs["base_url"],
104
+ azure_deployment=azure_configs["model_deployment"],
105
+ model=azure_configs["model_name"],
106
+ validate_base_url=False,
107
+ ))
108
+
109
+ # init the embeddings for answer_relevancy, answer_correctness and answer_similarity
110
+ evaluator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(
111
+ openai_api_version="2023-05-15",
112
+ azure_endpoint=azure_configs["base_url"],
113
+ azure_deployment=azure_configs["embedding_deployment"],
114
+ model=azure_configs["embedding_name"],
115
+ ))
116
+ ```
117
+
118
+ If you want more information on how to use other Azure services, please refer to the [langchain-azure](https://python.langchain.com/docs/integrations/chat/azure_chat_openai/) documentation.
119
+
120
+
121
+ === "Others"
122
+ If you are using a different LLM provider and using Langchain to interact with it, you can wrap your LLM in `LangchainLLMWrapper` so that it can be used with ragas.
123
+
124
+ ```python
125
+ from ragas.llms import LangchainLLMWrapper
126
+ evaluator_llm = LangchainLLMWrapper(your_llm_instance)
127
+ ```
128
+
129
+ For a more detailed guide, checkout [the guide on customizing models](../../howtos/customizations/customize_models/).
130
+
131
+ If you using LlamaIndex, you can use the `LlamaIndexLLMWrapper` to wrap your LLM so that it can be used with ragas.
132
+
133
+ ```python
134
+ from ragas.llms import LlamaIndexLLMWrapper
135
+ evaluator_llm = LlamaIndexLLMWrapper(your_llm_instance)
136
+ ```
137
+
138
+ For more information on how to use LlamaIndex, please refer to the [LlamaIndex Integration guide](../../howtos/integrations/_llamaindex/).
139
+
140
+ If your still not able use Ragas with your favorite LLM provider, please let us know by by commenting on this [issue](https://github.com/explodinggradients/ragas/issues/1617) and we'll add support for it 🙂.
@@ -0,0 +1,138 @@
1
+ === "OpenAI"
2
+ Install the langchain-openai package
3
+
4
+ ```bash
5
+ pip install langchain-openai
6
+ ```
7
+
8
+ then ensure you have your OpenAI key ready and available in your environment
9
+
10
+ ```python
11
+ import os
12
+ os.environ["OPENAI_API_KEY"] = "your-openai-key"
13
+ ```
14
+
15
+ Wrapp the LLMs in `LangchainLLMWrapper` so that it can be used with ragas.
16
+
17
+ ```python
18
+ from ragas.llms import LangchainLLMWrapper
19
+ from langchain_openai import ChatOpenAI
20
+ from langchain_openai import OpenAIEmbeddings
21
+ generator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
22
+ generator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
23
+ ```
24
+
25
+
26
+ === "Amazon Bedrock"
27
+ Install the langchain-aws package
28
+
29
+ ```bash
30
+ pip install langchain-aws
31
+ ```
32
+
33
+ then you have to set your AWS credentials and configurations
34
+
35
+ ```python
36
+ config = {
37
+ "credentials_profile_name": "your-profile-name", # E.g "default"
38
+ "region_name": "your-region-name", # E.g. "us-east-1"
39
+ "llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
40
+ "embeddings": "your-embedding-model-id", # E.g "amazon.titan-embed-text-v2:0"
41
+ "temperature": 0.4,
42
+ }
43
+ ```
44
+
45
+ define you LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
46
+
47
+ ```python
48
+ from langchain_aws import ChatBedrockConverse
49
+ from langchain_aws import BedrockEmbeddings
50
+ from ragas.llms import LangchainLLMWrapper
51
+ from ragas.embeddings import LangchainEmbeddingsWrapper
52
+
53
+ generator_llm = LangchainLLMWrapper(ChatBedrockConverse(
54
+ credentials_profile_name=config["credentials_profile_name"],
55
+ region_name=config["region_name"],
56
+ base_url=f"https://bedrock-runtime.{config['region_name']}.amazonaws.com",
57
+ model=config["llm"],
58
+ temperature=config["temperature"],
59
+ ))
60
+ generator_embeddings = LangchainEmbeddingsWrapper(BedrockEmbeddings(
61
+ credentials_profile_name=config["credentials_profile_name"],
62
+ region_name=config["region_name"],
63
+ model_id=config["embeddings"],
64
+ ))
65
+ ```
66
+
67
+ If you want more information on how to use other AWS services, please refer to the [langchain-aws](https://python.langchain.com/docs/integrations/providers/aws/) documentation.
68
+
69
+ === "Azure OpenAI"
70
+ Install the langchain-openai package
71
+
72
+ ```bash
73
+ pip install langchain-openai
74
+ ```
75
+
76
+ Ensure you have your Azure OpenAI key ready and available in your environment.
77
+
78
+ ```python
79
+ import os
80
+ os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-openai-key"
81
+
82
+ # other configuration
83
+ azure_config = {
84
+ "base_url": "", # your endpoint
85
+ "model_deployment": "", # your model deployment name
86
+ "model_name": "", # your model name
87
+ "embedding_deployment": "", # your embedding deployment name
88
+ "embedding_name": "", # your embedding name
89
+ }
90
+
91
+ ```
92
+
93
+ Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
94
+
95
+ ```python
96
+ from langchain_openai import AzureChatOpenAI
97
+ from langchain_openai import AzureOpenAIEmbeddings
98
+ from ragas.llms import LangchainLLMWrapper
99
+ from ragas.embeddings import LangchainEmbeddingsWrapper
100
+ generator_llm = LangchainLLMWrapper(AzureChatOpenAI(
101
+ openai_api_version="2023-05-15",
102
+ azure_endpoint=azure_configs["base_url"],
103
+ azure_deployment=azure_configs["model_deployment"],
104
+ model=azure_configs["model_name"],
105
+ validate_base_url=False,
106
+ ))
107
+
108
+ # init the embeddings for answer_relevancy, answer_correctness and answer_similarity
109
+ generator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(
110
+ openai_api_version="2023-05-15",
111
+ azure_endpoint=azure_configs["base_url"],
112
+ azure_deployment=azure_configs["embedding_deployment"],
113
+ model=azure_configs["embedding_name"],
114
+ ))
115
+ ```
116
+
117
+ If you want more information on how to use other Azure services, please refer to the [langchain-azure](https://python.langchain.com/docs/integrations/chat/azure_chat_openai/) documentation.
118
+
119
+ === "Others"
120
+ If you are using a different LLM provider and using Langchain to interact with it, you can wrap your LLM in `LangchainLLMWrapper` so that it can be used with ragas.
121
+
122
+ ```python
123
+ from ragas.llms import LangchainLLMWrapper
124
+ generator_llm = LangchainLLMWrapper(your_llm_instance)
125
+ ```
126
+
127
+ For a more detailed guide, checkout [the guide on customizing models](../../howtos/customizations/customize_models/).
128
+
129
+ If you using LlamaIndex, you can use the `LlamaIndexLLMWrapper` to wrap your LLM so that it can be used with ragas.
130
+
131
+ ```python
132
+ from ragas.llms import LlamaIndexLLMWrapper
133
+ generator_llm = LlamaIndexLLMWrapper(your_llm_instance)
134
+ ```
135
+
136
+ For more information on how to use LlamaIndex, please refer to the [LlamaIndex Integration guide](../../howtos/integrations/_llamaindex/).
137
+
138
+ If your still not able use Ragas with your favorite LLM provider, please let us know by by commenting on this [issue](https://github.com/explodinggradients/ragas/issues/1617) and we'll add support for it 🙂.
@@ -39,7 +39,7 @@ Now we will run the test generation using the loaded documents and the LLM setup
39
39
  ```python
40
40
  from ragas.testset import TestsetGenerator
41
41
 
42
- generator = TestsetGenerator(llm=generator_llm)
42
+ generator = TestsetGenerator(llm=generator_llm, embedding_model=generator_embeddings)
43
43
  dataset = generator.generate_with_langchain_docs(docs, testset_size=10)
44
44
  ```
45
45
 
@@ -141,9 +141,9 @@ query_distribution = default_query_distribution(generator_llm)
141
141
  ```
142
142
  ```
143
143
  [
144
- (AbstractQuerySynthesizer(llm=generator_llm), 0.25),
145
- (ComparativeAbstractQuerySynthesizer(llm=generator_llm), 0.25),
146
- (SpecificQuerySynthesizer(llm=generator_llm), 0.5),
144
+ (SingleHopSpecificQuerySynthesizer(llm=llm), 0.5),
145
+ (MultiHopAbstractQuerySynthesizer(llm=llm), 0.25),
146
+ (MultiHopSpecificQuerySynthesizer(llm=llm), 0.25),
147
147
  ]
148
148
  ```
149
149
 
@@ -109,7 +109,7 @@ from ragas.embeddings import LangchainEmbeddingsWrapper
109
109
  config = {
110
110
  "credentials_profile_name": "your-profile-name", # E.g "default"
111
111
  "region_name": "your-region-name", # E.g. "us-east-1"
112
- "llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20240620-v1:0"
112
+ "llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
113
113
  "embeddings": "your-embedding-model-id", # E.g "amazon.titan-embed-text-v2:0"
114
114
  "temperature": 0.4,
115
115
  }