ragas 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ragas-0.2.3 → ragas-0.2.4}/.readthedocs.yml +0 -1
- {ragas-0.2.3 → ragas-0.2.4}/Makefile +1 -4
- {ragas-0.2.3/src/ragas.egg-info → ragas-0.2.4}/PKG-INFO +3 -2
- {ragas-0.2.3 → ragas-0.2.4}/docs/community/index.md +11 -8
- ragas-0.2.4/docs/extra/components/choose_evaluator_llm.md +140 -0
- ragas-0.2.4/docs/extra/components/choose_generator_llm.md +138 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/rag_testset_generation.md +4 -4
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/customize_models.md +1 -1
- ragas-0.2.4/docs/howtos/customizations/metrics/_write_your_own_metric.md +298 -0
- ragas-0.2.3/docs/howtos/customizations/metrics/_write_your_own_metric.md → ragas-0.2.4/docs/howtos/customizations/metrics/_write_your_own_metric_advanced.md +4 -6
- ragas-0.2.4/docs/howtos/customizations/metrics/write_your_own_metric.ipynb +470 -0
- ragas-0.2.3/docs/howtos/customizations/metrics/write_your_own_metric.ipynb → ragas-0.2.4/docs/howtos/customizations/metrics/write_your_own_metric_advanced.ipynb +4 -6
- ragas-0.2.4/docs/howtos/integrations/_llamaindex.md +371 -0
- ragas-0.2.4/docs/howtos/integrations/llamaindex.ipynb +653 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/migrations/migrate_from_v01_to_v02.md +1 -1
- {ragas-0.2.3/scripts → ragas-0.2.4/docs}/ipynb_to_md.py +18 -16
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/testset_schema.md +2 -2
- {ragas-0.2.3 → ragas-0.2.4}/mkdocs.yml +10 -4
- {ragas-0.2.3 → ragas-0.2.4}/pyproject.toml +2 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/_version.py +2 -2
- ragas-0.2.4/src/ragas/async_utils.py +89 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/callbacks.py +13 -12
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/dataset_schema.py +53 -9
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/evaluation.py +18 -6
- ragas-0.2.4/src/ragas/executor.py +226 -0
- ragas-0.2.4/src/ragas/integrations/llama_index.py +104 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/llms/base.py +13 -5
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_answer_correctness.py +3 -3
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_answer_relevance.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_answer_similarity.py +9 -5
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_aspect_critic.py +10 -5
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_bleu_score.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_context_entities_recall.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_context_precision.py +5 -5
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_context_recall.py +2 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_datacompy_score.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_domain_specific_rubrics.py +29 -16
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_factual_correctness.py +1 -3
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_faithfulness.py +2 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_goal_accuracy.py +2 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_instance_specific_rubrics.py +2 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_multi_modal_faithfulness.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_multi_modal_relevance.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_noise_sensitivity.py +3 -4
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_rouge_score.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_simple_criteria.py +7 -7
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_sql_semantic_equivalence.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_string.py +5 -3
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_summarization.py +1 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_tool_call_accuracy.py +4 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/_topic_adherence.py +12 -16
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/base.py +40 -9
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/__init__.py +5 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/graph.py +84 -16
- ragas-0.2.4/src/ragas/testset/graph_queries.py +38 -0
- ragas-0.2.4/src/ragas/testset/persona.py +146 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/__init__.py +29 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/base.py +11 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/generate.py +166 -9
- ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/__init__.py +10 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/abstract.py +118 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/base.py +181 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/prompts.py +86 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/multi_hop/specific.py +116 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/prompts.py +49 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/__init__.py +3 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/base.py +145 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/prompts.py +35 -0
- ragas-0.2.4/src/ragas/testset/synthesizers/single_hop/specific.py +90 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/base.py +13 -15
- ragas-0.2.4/src/ragas/testset/transforms/default.py +93 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/engine.py +3 -1
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/__init__.py +2 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/llm_based.py +135 -45
- ragas-0.2.4/src/ragas/testset/transforms/relationship_builders/__init__.py +4 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/relationship_builders/cosine.py +0 -46
- ragas-0.2.4/src/ragas/testset/transforms/relationship_builders/traditional.py +155 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/splitters/headline.py +24 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/utils.py +20 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/validation.py +3 -2
- {ragas-0.2.3 → ragas-0.2.4/src/ragas.egg-info}/PKG-INFO +3 -2
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/SOURCES.txt +16 -4
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/requires.txt +2 -1
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/prompt/test_prompt_mixin.py +3 -3
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_analytics.py +5 -5
- ragas-0.2.4/tests/unit/test_async_utils.py +38 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_executor.py +43 -22
- ragas-0.2.4/tests/unit/test_executor_in_jupyter.ipynb +307 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_metric.py +33 -1
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_prompt.py +8 -8
- ragas-0.2.4/tests/unit/test_utils.py +89 -0
- ragas-0.2.3/docs/extra/components/choose_evaluator_llm.md +0 -68
- ragas-0.2.3/docs/extra/components/choose_generator_llm.md +0 -67
- ragas-0.2.3/docs/howtos/integrations/_llamaindex.md +0 -407
- ragas-0.2.3/docs/howtos/integrations/llamaindex.ipynb +0 -723
- ragas-0.2.3/src/ragas/async_utils.py +0 -49
- ragas-0.2.3/src/ragas/executor.py +0 -167
- ragas-0.2.3/src/ragas/integrations/llama_index.py +0 -101
- ragas-0.2.3/src/ragas/testset/synthesizers/__init__.py +0 -38
- ragas-0.2.3/src/ragas/testset/synthesizers/abstract_query.py +0 -349
- ragas-0.2.3/src/ragas/testset/synthesizers/base_query.py +0 -95
- ragas-0.2.3/src/ragas/testset/synthesizers/prompts.py +0 -408
- ragas-0.2.3/src/ragas/testset/synthesizers/specific_query.py +0 -115
- ragas-0.2.3/src/ragas/testset/transforms/default.py +0 -80
- ragas-0.2.3/src/ragas/testset/transforms/relationship_builders/__init__.py +0 -3
- ragas-0.2.3/tests/unit/test_executor_in_jupyter.ipynb +0 -190
- ragas-0.2.3/tests/unit/test_utils.py +0 -53
- {ragas-0.2.3 → ragas-0.2.4}/.dockerignore +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.gitattributes +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.github/ISSUE_TEMPLATE/question.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.github/workflows/ci.yaml +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.github/workflows/issue-manager.yaml +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.github/workflows/python-publish.yml +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/.gitignore +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/DEVELOPMENT.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/LICENSE +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/README.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/Makefile +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_ipython3.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_ipython3_dark.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_ipython3_light.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_python.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_python_dark.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/highlight_python_light.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/css/ragas.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/favicon.ico +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/bar-graph.svg +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-emb-results.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-embeddings.jpeg +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-llm-result.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-llms-front.jpeg +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/compare-llms-testset.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/component-wise-metrics.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/eval-evolve.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/kg_rag.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/langsmith-tracing-faithfullness.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/langsmith-tracing-overview.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/logo-black.svg +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/logo.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/metrics_mindmap.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/question_types.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/quickstart-output.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/ragas-logo.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/ragas_workflow_white.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/scenario_rag.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/testset_output.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/imgs/trace-langsmith.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/header_border.js +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/mathjax.js +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/mendable_chat_bubble.js +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/octolane.js +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/js/toggle.js +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-dataset.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-evaluation.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-feedback.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/langsmith-ragas-chain-trace.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/list-score-traces-ragas.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/_static/traces-score-ragas.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/alfred.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/eval_dataset.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/eval_sample.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/components/prompt.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/feedback/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/agents.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/answer_correctness.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/answer_relevance.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/aspect_critic.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/context_entities_recall.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/context_precision.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/context_recall.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/factual_correctness.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/faithfulness.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/general_purpose.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/multi_modal_faithfulness.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/multi_modal_relevance.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/noise_sensitivity.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/rubrics_based.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/semantic_similarity.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/sql.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/summarization_score.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/traditional.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/metrics/overview/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/test_data_generation/agents.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/test_data_generation/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/concepts/test_data_generation/rag.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/conf.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/extra/fonts/Satoshi-Variable.ttf +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/extra/fonts/Satoshi-VariableItalic.ttf +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/extra/style.css +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/install.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/rag_evaluation.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/raga_evaluation_output.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/getstarted/testset_output.png +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/_cost.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/add_to_ci.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/compare_embeddings.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/compare_llms.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/cost.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/applications/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/_run_config.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/_metrics_language_adaptation.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/_modifying-prompts-metrics.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/metrics_language_adaptation.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/modifying-prompts-metrics.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/metrics/tracing.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/run_config.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/customizations/testgenerator/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_arize.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_athina.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_haystack.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_helicone.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_langchain.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_langfuse.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_langsmith.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_openlayer.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_opik.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_tonic-validate.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/_zeno.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/arize.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/athina.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/haystack.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/helicone.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/langchain.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/langfuse.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/langsmith.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/nyc_wikipedia/nyc_text.txt +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/openlayer.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/opik.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/tonic-validate.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/howtos/integrations/zeno.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/index.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/make.bat +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/embeddings.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/evaluate.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/evaluation_schema.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/executor.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/generate.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/graph.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/integrations.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/llms.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/metrics.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/prompt.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/run_config.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/synthesizers.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/docs/references/transforms.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/references.md +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/requirements/dev.txt +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/requirements/docs.txt +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/requirements/test.txt +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/setup.cfg +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/_analytics.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/cost.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/embeddings/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/embeddings/base.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/exceptions.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/experimental/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/helicone.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/langchain.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/langsmith.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/integrations/opik.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/llms/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/messages.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/metrics/utils.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/base.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/mixin.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/multi_modal_prompt.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/pydantic_prompt.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/prompt/utils.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/run_config.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/testset_schema.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/synthesizers/utils.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/embeddings.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/extractors/regex_based.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas/testset/transforms/splitters/__init__.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/dependency_links.txt +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/src/ragas.egg-info/top_level.txt +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/Dockerfile +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/benchmark_eval.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/benchmark_testsetgen.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/benchmarks/utils.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/conftest.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_adaptation.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_amnesty_in_ci.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_evaluation_in_jupyter.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_fullflow.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_langsmith.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/e2e/test_testset_gen_in_jupyter.ipynb +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/llms/test_llm.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/prompt/test_prompt_utils.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_cost.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_dataset_schema.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_embeddings.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_import.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_run_config.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_simple.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_testset_schema.py +0 -0
- {ragas-0.2.3 → ragas-0.2.4}/tests/unit/test_validation.py +0 -0
|
@@ -39,11 +39,8 @@ run-ci: format lint type test ## Running all CI checks
|
|
|
39
39
|
rewrite-docs: ## Use GPT4 to rewrite the documentation
|
|
40
40
|
@echo "Rewriting the documentation in directory $(DIR)..."
|
|
41
41
|
$(Q)python $(GIT_ROOT)/docs/python alphred.py --directory $(DIR)
|
|
42
|
-
ipynb-to-md: ## Convert ipynb files to md files
|
|
43
|
-
$(Q)python $(GIT_ROOT)/scripts/ipynb_to_md.py
|
|
44
42
|
docsite: ## Build and serve documentation
|
|
45
|
-
$(Q)
|
|
46
|
-
$(Q)mkdocs serve
|
|
43
|
+
$(Q)mkdocs serve --dirtyreload
|
|
47
44
|
|
|
48
45
|
# Benchmarks
|
|
49
46
|
run-benchmarks-eval: ## Run benchmarks for Evaluation
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: ragas
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Description-Content-Type: text/markdown
|
|
5
5
|
License-File: LICENSE
|
|
6
6
|
Requires-Dist: numpy
|
|
@@ -23,10 +23,11 @@ Requires-Dist: rouge_score; extra == "all"
|
|
|
23
23
|
Requires-Dist: rapidfuzz; extra == "all"
|
|
24
24
|
Requires-Dist: pandas; extra == "all"
|
|
25
25
|
Requires-Dist: datacompy; extra == "all"
|
|
26
|
+
Requires-Dist: llama_index; extra == "all"
|
|
26
27
|
Provides-Extra: docs
|
|
27
28
|
Requires-Dist: mkdocs>=1.6.1; extra == "docs"
|
|
28
29
|
Requires-Dist: mkdocs-material; extra == "docs"
|
|
29
|
-
Requires-Dist: mkdocs-
|
|
30
|
+
Requires-Dist: mkdocs-material[imaging]; extra == "docs"
|
|
30
31
|
Requires-Dist: mkdocstrings[python]; extra == "docs"
|
|
31
32
|
Requires-Dist: mkdocs-glightbox; extra == "docs"
|
|
32
33
|
Requires-Dist: mkdocs-autorefs; extra == "docs"
|
|
@@ -17,8 +17,8 @@ Meet some of our outstanding members who made significant contributions !
|
|
|
17
17
|
Explore insightful articles, tutorials, and stories written by and for our community members.
|
|
18
18
|
|
|
19
19
|
- [Luka Panić](https://www.linkedin.com/in/luka-pani%C4%87-20b671277/) shares his work on
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
- [Ragas Evaluation: In-Depth Insights | PIXION Blog](https://pixion.co/blog/ragas-evaluation-in-depth-insights): A detailed explanation of the metrics and how they are calculated.
|
|
21
|
+
- [RAG in practice - Test Set Generation | PIXION Blog](https://pixion.co/blog/rag-in-practice-test-set-generation): A tutorial on how to generate a test set using Ragas.
|
|
22
22
|
- [Shanthi Vardhan](https://www.linkedin.com/in/shanthivardhan/) shares how his team at [Atomicwork uses ragas](https://www.atomicwork.com/blog/ragas-improving-atom-accuracy) to improve their AI system's ability to accurately identify and retrieve more precise information for enhanced service management.
|
|
23
23
|
- [Pinecone's](https://pinecone.io/blog) study on how RAGs can enhance capabilities of LLMs in ["RAG makes LLMs better and equal"](https://www.pinecone.io/blog/rag-study/) uses ragas to proves context retrieval makes LLMs provide significantly better results, even when increasing the data size to 1 billion.
|
|
24
24
|
- [Aishwarya Prabhat](https://www.linkedin.com/in/aishwaryaprabhat/) shares her expertise on advanced RAG techniques in her comprehensive guide, ["Performing, Evaluating & Tracking Advanced RAG (ft. AzureML, LlamaIndex & Ragas)"](https://www.linkedin.com/pulse/performing-evaluating-tracking-advanced-rag-ft-azureml-prabhat-i1rkc/).
|
|
@@ -34,13 +34,16 @@ Explore insightful articles, tutorials, and stories written by and for our commu
|
|
|
34
34
|
- [RAG Evaluation : Computational Metrics in RAG and Calculation Methods in Ragas](https://tech.beatrust.com/entry/2024/05/02/RAG_Evaluation_%3A_Computational_Metrics_in_RAG_and_Calculation_Methods_in_Ragas)
|
|
35
35
|
- [RAG Evaluation: Assessing the Usefulness of Ragas](https://tech.beatrust.com/entry/2024/05/02/RAG_Evaluation%3A_Assessing_the_Usefulness_of_Ragas)
|
|
36
36
|
- [Atita Arora](https://www.linkedin.com/in/atitaarora/) writes about [Evaluating Retrieval Augmented Generation using RAGAS](https://superlinked.com/vectorhub/articles/retrieval-augmented-generation-eval-qdrant-ragas), an end-to-end tutorial on building RAG using [Qdrant](https://qdrant.tech/) and [Langchain](https://www.langchain.com/) and evaluating it with RAGAS.
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
- *Bonus content* : Learn how to create an evaluation dataset that serves as a reference point for evaluating our RAG pipeline, Understand the RAGAS evaluation metrics and how to make sense of them and putting them in action to test a Naive RAG pipeline and measure its performance using RAGAS metrics.
|
|
38
|
+
- *Code walkthrough* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas
|
|
39
|
+
- *Code walkthrough using [Deepset Haystack](https://haystack.deepset.ai/) and [Mixedbread.ai](https://www.mixedbread.ai/)* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas-haystack
|
|
40
40
|
- [Minoru Onda](https://x.com/minorun365) writes for beginners about how to start Ragas v0.2 evaluation with Amazon Bedrock, and integrate with Langfuse.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
- [RAG精度評価の定番ツール「Ragas」にAWSのBedrockで入門しよう!(v0.2対応) - Qiita](https://qiita.com/minorun365/items/2f4e238f8bbc6e393ba5)
|
|
42
|
+
- [生成AIアプリの出力をRagasで評価して、LangfuseでGUI監視しよう! - Qiita](https://qiita.com/minorun365/items/70ad2f5a0afaac6e5cb9)
|
|
43
|
+
- [Yunnglin](https://github.com/Yunnglin) has penned a guide on integrating Ragas v0.2 into [EvalScope](https://github.com/modelscope/eval-scope) (an evaluation framework for large models), thereby utilizing the [ModelScope](https://github.com/modelscope/modelscope) ecosystem.
|
|
44
|
+
- Tutorial: [Using Ragas with EvalScope](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html)
|
|
45
|
+
- 教程: [在EvalScope中使用Ragas一键发起RAG评估](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/backend/rageval_backend/ragas.html)
|
|
46
|
+
- 最佳实践: [打破文本边界:如何进行多模态RAG评估](https://evalscope.readthedocs.io/zh-cn/latest/blog/RAG/multimodal_RAG.html#multimodal-rag)
|
|
44
47
|
|
|
45
48
|
## **📅 Events**
|
|
46
49
|
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
|
|
2
|
+
=== "OpenAI"
|
|
3
|
+
Install the langchain-openai package
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install langchain-openai
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Ensure you have your OpenAI key ready and available in your environment.
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
import os
|
|
13
|
+
os.environ["OPENAI_API_KEY"] = "your-openai-key"
|
|
14
|
+
```
|
|
15
|
+
Wrap the LLMs in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from ragas.llms import LangchainLLMWrapper
|
|
19
|
+
from ragas.embeddings import LangchainEmbeddingsWrapper
|
|
20
|
+
from langchain_openai import ChatOpenAI
|
|
21
|
+
from langchain_openai import OpenAIEmbeddings
|
|
22
|
+
evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
|
|
23
|
+
evaluator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
=== "Amazon Bedrock"
|
|
28
|
+
Install the langchain-aws package
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install langchain-aws
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Then you have to set your AWS credentials and configurations
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
config = {
|
|
38
|
+
"credentials_profile_name": "your-profile-name", # E.g "default"
|
|
39
|
+
"region_name": "your-region-name", # E.g. "us-east-1"
|
|
40
|
+
"llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
|
41
|
+
"embeddings": "your-embedding-model-id", # E.g "amazon.titan-embed-text-v2:0"
|
|
42
|
+
"temperature": 0.4,
|
|
43
|
+
}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from langchain_aws import ChatBedrockConverse
|
|
50
|
+
from langchain_aws import BedrockEmbeddings
|
|
51
|
+
from ragas.llms import LangchainLLMWrapper
|
|
52
|
+
from ragas.embeddings import LangchainEmbeddingsWrapper
|
|
53
|
+
|
|
54
|
+
evaluator_llm = LangchainLLMWrapper(ChatBedrockConverse(
|
|
55
|
+
credentials_profile_name=config["credentials_profile_name"],
|
|
56
|
+
region_name=config["region_name"],
|
|
57
|
+
base_url=f"https://bedrock-runtime.{config['region_name']}.amazonaws.com",
|
|
58
|
+
model=config["llm"],
|
|
59
|
+
temperature=config["temperature"],
|
|
60
|
+
))
|
|
61
|
+
evaluator_embeddings = LangchainEmbeddingsWrapper(BedrockEmbeddings(
|
|
62
|
+
credentials_profile_name=config["credentials_profile_name"],
|
|
63
|
+
region_name=config["region_name"],
|
|
64
|
+
model_id=config["embeddings"],
|
|
65
|
+
))
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
If you want more information on how to use other AWS services, please refer to the [langchain-aws](https://python.langchain.com/docs/integrations/providers/aws/) documentation.
|
|
69
|
+
|
|
70
|
+
=== "Azure OpenAI"
|
|
71
|
+
Install the langchain-openai package
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install langchain-openai
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Ensure you have your Azure OpenAI key ready and available in your environment.
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
import os
|
|
81
|
+
os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-openai-key"
|
|
82
|
+
|
|
83
|
+
# other configuration
|
|
84
|
+
azure_config = {
|
|
85
|
+
"base_url": "", # your endpoint
|
|
86
|
+
"model_deployment": "", # your model deployment name
|
|
87
|
+
"model_name": "", # your model name
|
|
88
|
+
"embedding_deployment": "", # your embedding deployment name
|
|
89
|
+
"embedding_name": "", # your embedding name
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from langchain_openai import AzureChatOpenAI
|
|
98
|
+
from langchain_openai import AzureOpenAIEmbeddings
|
|
99
|
+
from ragas.llms import LangchainLLMWrapper
|
|
100
|
+
from ragas.embeddings import LangchainEmbeddingsWrapper
|
|
101
|
+
evaluator_llm = LangchainLLMWrapper(AzureChatOpenAI(
|
|
102
|
+
openai_api_version="2023-05-15",
|
|
103
|
+
azure_endpoint=azure_configs["base_url"],
|
|
104
|
+
azure_deployment=azure_configs["model_deployment"],
|
|
105
|
+
model=azure_configs["model_name"],
|
|
106
|
+
validate_base_url=False,
|
|
107
|
+
))
|
|
108
|
+
|
|
109
|
+
# init the embeddings for answer_relevancy, answer_correctness and answer_similarity
|
|
110
|
+
evaluator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(
|
|
111
|
+
openai_api_version="2023-05-15",
|
|
112
|
+
azure_endpoint=azure_configs["base_url"],
|
|
113
|
+
azure_deployment=azure_configs["embedding_deployment"],
|
|
114
|
+
model=azure_configs["embedding_name"],
|
|
115
|
+
))
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
If you want more information on how to use other Azure services, please refer to the [langchain-azure](https://python.langchain.com/docs/integrations/chat/azure_chat_openai/) documentation.
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
=== "Others"
|
|
122
|
+
If you are using a different LLM provider and using Langchain to interact with it, you can wrap your LLM in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from ragas.llms import LangchainLLMWrapper
|
|
126
|
+
evaluator_llm = LangchainLLMWrapper(your_llm_instance)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
For a more detailed guide, checkout [the guide on customizing models](../../howtos/customizations/customize_models/).
|
|
130
|
+
|
|
131
|
+
If you using LlamaIndex, you can use the `LlamaIndexLLMWrapper` to wrap your LLM so that it can be used with ragas.
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from ragas.llms import LlamaIndexLLMWrapper
|
|
135
|
+
evaluator_llm = LlamaIndexLLMWrapper(your_llm_instance)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
For more information on how to use LlamaIndex, please refer to the [LlamaIndex Integration guide](../../howtos/integrations/_llamaindex/).
|
|
139
|
+
|
|
140
|
+
If your still not able use Ragas with your favorite LLM provider, please let us know by by commenting on this [issue](https://github.com/explodinggradients/ragas/issues/1617) and we'll add support for it 🙂.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
=== "OpenAI"
|
|
2
|
+
Install the langchain-openai package
|
|
3
|
+
|
|
4
|
+
```bash
|
|
5
|
+
pip install langchain-openai
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
then ensure you have your OpenAI key ready and available in your environment
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
import os
|
|
12
|
+
os.environ["OPENAI_API_KEY"] = "your-openai-key"
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Wrapp the LLMs in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from ragas.llms import LangchainLLMWrapper
|
|
19
|
+
from langchain_openai import ChatOpenAI
|
|
20
|
+
from langchain_openai import OpenAIEmbeddings
|
|
21
|
+
generator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
|
|
22
|
+
generator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
=== "Amazon Bedrock"
|
|
27
|
+
Install the langchain-aws package
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install langchain-aws
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
then you have to set your AWS credentials and configurations
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
config = {
|
|
37
|
+
"credentials_profile_name": "your-profile-name", # E.g "default"
|
|
38
|
+
"region_name": "your-region-name", # E.g. "us-east-1"
|
|
39
|
+
"llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
|
40
|
+
"embeddings": "your-embedding-model-id", # E.g "amazon.titan-embed-text-v2:0"
|
|
41
|
+
"temperature": 0.4,
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
define you LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from langchain_aws import ChatBedrockConverse
|
|
49
|
+
from langchain_aws import BedrockEmbeddings
|
|
50
|
+
from ragas.llms import LangchainLLMWrapper
|
|
51
|
+
from ragas.embeddings import LangchainEmbeddingsWrapper
|
|
52
|
+
|
|
53
|
+
generator_llm = LangchainLLMWrapper(ChatBedrockConverse(
|
|
54
|
+
credentials_profile_name=config["credentials_profile_name"],
|
|
55
|
+
region_name=config["region_name"],
|
|
56
|
+
base_url=f"https://bedrock-runtime.{config['region_name']}.amazonaws.com",
|
|
57
|
+
model=config["llm"],
|
|
58
|
+
temperature=config["temperature"],
|
|
59
|
+
))
|
|
60
|
+
generator_embeddings = LangchainEmbeddingsWrapper(BedrockEmbeddings(
|
|
61
|
+
credentials_profile_name=config["credentials_profile_name"],
|
|
62
|
+
region_name=config["region_name"],
|
|
63
|
+
model_id=config["embeddings"],
|
|
64
|
+
))
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
If you want more information on how to use other AWS services, please refer to the [langchain-aws](https://python.langchain.com/docs/integrations/providers/aws/) documentation.
|
|
68
|
+
|
|
69
|
+
=== "Azure OpenAI"
|
|
70
|
+
Install the langchain-openai package
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install langchain-openai
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Ensure you have your Azure OpenAI key ready and available in your environment.
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
import os
|
|
80
|
+
os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-openai-key"
|
|
81
|
+
|
|
82
|
+
# other configuration
|
|
83
|
+
azure_config = {
|
|
84
|
+
"base_url": "", # your endpoint
|
|
85
|
+
"model_deployment": "", # your model deployment name
|
|
86
|
+
"model_name": "", # your model name
|
|
87
|
+
"embedding_deployment": "", # your embedding deployment name
|
|
88
|
+
"embedding_name": "", # your embedding name
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from langchain_openai import AzureChatOpenAI
|
|
97
|
+
from langchain_openai import AzureOpenAIEmbeddings
|
|
98
|
+
from ragas.llms import LangchainLLMWrapper
|
|
99
|
+
from ragas.embeddings import LangchainEmbeddingsWrapper
|
|
100
|
+
generator_llm = LangchainLLMWrapper(AzureChatOpenAI(
|
|
101
|
+
openai_api_version="2023-05-15",
|
|
102
|
+
azure_endpoint=azure_configs["base_url"],
|
|
103
|
+
azure_deployment=azure_configs["model_deployment"],
|
|
104
|
+
model=azure_configs["model_name"],
|
|
105
|
+
validate_base_url=False,
|
|
106
|
+
))
|
|
107
|
+
|
|
108
|
+
# init the embeddings for answer_relevancy, answer_correctness and answer_similarity
|
|
109
|
+
generator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(
|
|
110
|
+
openai_api_version="2023-05-15",
|
|
111
|
+
azure_endpoint=azure_configs["base_url"],
|
|
112
|
+
azure_deployment=azure_configs["embedding_deployment"],
|
|
113
|
+
model=azure_configs["embedding_name"],
|
|
114
|
+
))
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
If you want more information on how to use other Azure services, please refer to the [langchain-azure](https://python.langchain.com/docs/integrations/chat/azure_chat_openai/) documentation.
|
|
118
|
+
|
|
119
|
+
=== "Others"
|
|
120
|
+
If you are using a different LLM provider and using Langchain to interact with it, you can wrap your LLM in `LangchainLLMWrapper` so that it can be used with ragas.
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
from ragas.llms import LangchainLLMWrapper
|
|
124
|
+
generator_llm = LangchainLLMWrapper(your_llm_instance)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
For a more detailed guide, checkout [the guide on customizing models](../../howtos/customizations/customize_models/).
|
|
128
|
+
|
|
129
|
+
If you using LlamaIndex, you can use the `LlamaIndexLLMWrapper` to wrap your LLM so that it can be used with ragas.
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
from ragas.llms import LlamaIndexLLMWrapper
|
|
133
|
+
generator_llm = LlamaIndexLLMWrapper(your_llm_instance)
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
For more information on how to use LlamaIndex, please refer to the [LlamaIndex Integration guide](../../howtos/integrations/_llamaindex/).
|
|
137
|
+
|
|
138
|
+
If your still not able use Ragas with your favorite LLM provider, please let us know by by commenting on this [issue](https://github.com/explodinggradients/ragas/issues/1617) and we'll add support for it 🙂.
|
|
@@ -39,7 +39,7 @@ Now we will run the test generation using the loaded documents and the LLM setup
|
|
|
39
39
|
```python
|
|
40
40
|
from ragas.testset import TestsetGenerator
|
|
41
41
|
|
|
42
|
-
generator = TestsetGenerator(llm=generator_llm)
|
|
42
|
+
generator = TestsetGenerator(llm=generator_llm, embedding_model=generator_embeddings)
|
|
43
43
|
dataset = generator.generate_with_langchain_docs(docs, testset_size=10)
|
|
44
44
|
```
|
|
45
45
|
|
|
@@ -141,9 +141,9 @@ query_distribution = default_query_distribution(generator_llm)
|
|
|
141
141
|
```
|
|
142
142
|
```
|
|
143
143
|
[
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
144
|
+
(SingleHopSpecificQuerySynthesizer(llm=llm), 0.5),
|
|
145
|
+
(MultiHopAbstractQuerySynthesizer(llm=llm), 0.25),
|
|
146
|
+
(MultiHopSpecificQuerySynthesizer(llm=llm), 0.25),
|
|
147
147
|
]
|
|
148
148
|
```
|
|
149
149
|
|
|
@@ -109,7 +109,7 @@ from ragas.embeddings import LangchainEmbeddingsWrapper
|
|
|
109
109
|
config = {
|
|
110
110
|
"credentials_profile_name": "your-profile-name", # E.g "default"
|
|
111
111
|
"region_name": "your-region-name", # E.g. "us-east-1"
|
|
112
|
-
"llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-
|
|
112
|
+
"llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
|
113
113
|
"embeddings": "your-embedding-model-id", # E.g "amazon.titan-embed-text-v2:0"
|
|
114
114
|
"temperature": 0.4,
|
|
115
115
|
}
|