langchain 0.3.26__py3-none-any.whl → 0.3.27__py3-none-any.whl

langchain/evaluation/comparison/eval_chain.py

@@ -11,6 +11,7 @@ from langchain_core.language_models import BaseLanguageModel
 from langchain_core.output_parsers import BaseOutputParser
 from langchain_core.prompts.prompt import PromptTemplate
 from pydantic import ConfigDict, Field
+from typing_extensions import override
 
 from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
 from langchain.chains.llm import LLMChain
@@ -69,7 +70,7 @@ def resolve_pairwise_criteria(
             Criteria.DEPTH,
         ]
         return {k.value: _SUPPORTED_CRITERIA[k] for k in _default_criteria}
-    elif isinstance(criteria, Criteria):
+    if isinstance(criteria, Criteria):
         criteria_ = {criteria.value: _SUPPORTED_CRITERIA[criteria]}
     elif isinstance(criteria, str):
         if criteria in _SUPPORTED_CRITERIA:
@@ -86,11 +87,12 @@ def resolve_pairwise_criteria(
         }
     else:
         if not criteria:
-            raise ValueError(
+            msg = (
                 "Criteria cannot be empty. "
                 "Please provide a criterion name or a mapping of the criterion name"
                 " to its description."
             )
+            raise ValueError(msg)
         criteria_ = dict(criteria)
     return criteria_
 
@@ -132,11 +134,12 @@ class PairwiseStringResultOutputParser(BaseOutputParser[dict]):
             verdict = match.group(1)
 
         if not match or verdict not in {"A", "B", "C"}:
-            raise ValueError(
+            msg = (
                 f"Invalid output: {text}. "
                 "Output must contain a double bracketed string\
                  with the verdict 'A', 'B', or 'C'."
             )
+            raise ValueError(msg)
         # C means the models are tied. Return 'None' meaning no preference
         verdict_ = None if verdict == "C" else verdict
         score = {
@@ -184,7 +187,7 @@ class PairwiseStringEvalChain(PairwiseStringEvaluator, LLMEvalChain, LLMChain):
 
     output_key: str = "results"  #: :meta private:
     output_parser: BaseOutputParser = Field(
-        default_factory=PairwiseStringResultOutputParser
+        default_factory=PairwiseStringResultOutputParser,
     )
 
     @classmethod
@@ -256,16 +259,17 @@ class PairwiseStringEvalChain(PairwiseStringEvaluator, LLMEvalChain, LLMChain):
         if not hasattr(llm, "model_name") or not llm.model_name.startswith("gpt-4"):
             logger.warning(
                 "This chain was only tested with GPT-4. \
-Performance may be significantly worse with other models."
+Performance may be significantly worse with other models.",
             )
 
         expected_input_vars = {"prediction", "prediction_b", "input", "criteria"}
         prompt_ = prompt or COMPARISON_TEMPLATE.partial(reference="")
         if expected_input_vars != set(prompt_.input_variables):
-            raise ValueError(
+            msg = (
                 f"Input variables should be {expected_input_vars}, "
                 f"but got {prompt_.input_variables}"
             )
+            raise ValueError(msg)
         criteria_ = resolve_pairwise_criteria(criteria)
         criteria_str = "\n".join(f"{k}: {v}" if v else k for k, v in criteria_.items())
         criteria_str = CRITERIA_INSTRUCTIONS + criteria_str if criteria_str else ""
@@ -275,7 +279,7 @@ Performance may be significantly worse with other models."
         self,
         prediction: str,
         prediction_b: str,
-        input: Optional[str],
+        input_: Optional[str],
         reference: Optional[str],
     ) -> dict:
         """Prepare the input for the chain.
@@ -283,21 +287,21 @@ Performance may be significantly worse with other models."
         Args:
             prediction (str): The output string from the first model.
             prediction_b (str): The output string from the second model.
-            input (str, optional): The input or task string.
+            input_ (str, optional): The input or task string.
             reference (str, optional): The reference string, if any.
 
         Returns:
             dict: The prepared input for the chain.
 
         """
-        input_ = {
+        input_dict = {
             "prediction": prediction,
             "prediction_b": prediction_b,
-            "input": input,
+            "input": input_,
         }
         if self.requires_reference:
-            input_["reference"] = reference
-        return input_
+            input_dict["reference"] = reference
+        return input_dict
 
     def _prepare_output(self, result: dict) -> dict:
         """Prepare the output."""
@@ -306,6 +310,7 @@ Performance may be significantly worse with other models."
             parsed[RUN_KEY] = result[RUN_KEY]
         return parsed
 
+    @override
     def _evaluate_string_pairs(
         self,
         *,
@@ -348,6 +353,7 @@ Performance may be significantly worse with other models."
         )
         return self._prepare_output(result)
 
+    @override
     async def _aevaluate_string_pairs(
         self,
         *,
@@ -444,10 +450,11 @@ class LabeledPairwiseStringEvalChain(PairwiseStringEvalChain):
         }
         prompt_ = prompt or COMPARISON_TEMPLATE_WITH_REFERENCE
         if expected_input_vars != set(prompt_.input_variables):
-            raise ValueError(
+            msg = (
                 f"Input variables should be {expected_input_vars}, "
                 f"but got {prompt_.input_variables}"
             )
+            raise ValueError(msg)
         criteria_ = resolve_pairwise_criteria(criteria)
         criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
         criteria_str = CRITERIA_INSTRUCTIONS + criteria_str if criteria_str else ""

langchain/evaluation/comparison/prompt.py

@@ -3,9 +3,8 @@
 This prompt is used to compare two responses and evaluate which one best follows the instructions
 and answers the question. The prompt is based on the paper from
 Zheng, et. al. https://arxiv.org/abs/2306.05685
-"""
+"""  # noqa: E501
 
-# flake8: noqa
 from langchain_core.prompts.chat import ChatPromptTemplate
 
 SYSTEM_MESSAGE = 'Please act as an impartial judge and evaluate the quality \

langchain/evaluation/criteria/__init__.py

@@ -53,4 +53,4 @@ from langchain.evaluation.criteria.eval_chain import (
     LabeledCriteriaEvalChain,
 )
 
-__all__ = ["CriteriaEvalChain", "LabeledCriteriaEvalChain", "Criteria"]
+__all__ = ["Criteria", "CriteriaEvalChain", "LabeledCriteriaEvalChain"]

langchain/evaluation/criteria/eval_chain.py

@@ -10,6 +10,7 @@ from langchain_core.language_models import BaseLanguageModel
 from langchain_core.output_parsers import BaseOutputParser
 from langchain_core.prompts import BasePromptTemplate
 from pydantic import ConfigDict, Field
+from typing_extensions import override
 
 from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
 from langchain.chains.llm import LLMChain
@@ -156,11 +157,12 @@ def resolve_criteria(
         criteria_ = {criteria.name: criteria.critique_request}
     else:
         if not criteria:
-            raise ValueError(
+            msg = (
                 "Criteria cannot be empty. "
                 "Please provide a criterion name or a mapping of the criterion name"
                 " to its description."
             )
+            raise ValueError(msg)
         criteria_ = dict(criteria)
     return criteria_
 
@@ -271,15 +273,17 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
 
     @classmethod
     def _resolve_prompt(
-        cls, prompt: Optional[BasePromptTemplate] = None
+        cls,
+        prompt: Optional[BasePromptTemplate] = None,
     ) -> BasePromptTemplate:
         expected_input_vars = {"input", "output", "criteria"}
         prompt_ = prompt or PROMPT
         if expected_input_vars != set(prompt_.input_variables):
-            raise ValueError(
+            msg = (
                 f"Input variables should be {expected_input_vars}, "
                 f"but got {prompt_.input_variables}"
             )
+            raise ValueError(msg)
         return prompt_
 
     @classmethod
@@ -360,12 +364,13 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
         """
         prompt_ = cls._resolve_prompt(prompt)
         if criteria == Criteria.CORRECTNESS:
-            raise ValueError(
+            msg = (
                 "Correctness should not be used in the reference-free"
                 " 'criteria' evaluator (CriteriaEvalChain)."
                 " Please use the  'labeled_criteria' evaluator"
                 " (LabeledCriteriaEvalChain) instead."
             )
+            raise ValueError(msg)
         criteria_ = cls.resolve_criteria(criteria)
         criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
         prompt_ = prompt_.partial(criteria=criteria_str)
@@ -380,16 +385,16 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
         self,
         prediction: str,
         reference: Optional[str],
-        input: Optional[str],
+        input_: Optional[str],
     ) -> dict:
         """Get the evaluation input."""
-        input_ = {
-            "input": input,
+        input_dict = {
+            "input": input_,
             "output": prediction,
         }
         if self.requires_reference:
-            input_["reference"] = reference
-        return input_
+            input_dict["reference"] = reference
+        return input_dict
 
     def _prepare_output(self, result: dict) -> dict:
         """Prepare the output."""
@@ -398,6 +403,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
             parsed[RUN_KEY] = result[RUN_KEY]
         return parsed
 
+    @override
     def _evaluate_strings(
         self,
         *,
@@ -453,6 +459,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
         )
         return self._prepare_output(result)
 
+    @override
     async def _aevaluate_strings(
         self,
         *,
@@ -523,15 +530,17 @@ class LabeledCriteriaEvalChain(CriteriaEvalChain):
 
     @classmethod
     def _resolve_prompt(
-        cls, prompt: Optional[BasePromptTemplate] = None
+        cls,
+        prompt: Optional[BasePromptTemplate] = None,
     ) -> BasePromptTemplate:
         expected_input_vars = {"input", "output", "criteria", "reference"}
         prompt_ = prompt or PROMPT_WITH_REFERENCES
         if expected_input_vars != set(prompt_.input_variables):
-            raise ValueError(
+            msg = (
                 f"Input variables should be {expected_input_vars}, "
                 f"but got {prompt_.input_variables}"
             )
+            raise ValueError(msg)
         return prompt_
 
     @classmethod

langchain/evaluation/criteria/prompt.py

@@ -1,4 +1,3 @@
-# flake8: noqa
 # Credit to https://github.com/openai/evals/tree/main
 
 from langchain_core.prompts import PromptTemplate
@@ -13,7 +12,7 @@ template = """You are assessing a submitted answer on a given task or input base
 [Criteria]: {criteria}
 ***
 [END DATA]
-Does the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line."""
+Does the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line."""  # noqa: E501
 
 PROMPT = PromptTemplate(
     input_variables=["input", "output", "criteria"], template=template
@@ -31,7 +30,7 @@ template = """You are assessing a submitted answer on a given task or input base
 [Reference]: {reference}
 ***
 [END DATA]
-Does the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line."""
+Does the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line."""  # noqa: E501
 
 PROMPT_WITH_REFERENCES = PromptTemplate(
     input_variables=["input", "output", "criteria", "reference"], template=template

langchain/evaluation/embedding_distance/base.py

@@ -23,12 +23,10 @@ from langchain.schema import RUN_KEY
 def _import_numpy() -> Any:
     try:
         import numpy as np
-
-        return np
     except ImportError as e:
-        raise ImportError(
-            "Could not import numpy, please install with `pip install numpy`."
-        ) from e
+        msg = "Could not import numpy, please install with `pip install numpy`."
+        raise ImportError(msg) from e
+    return np
 
 
 logger = logging.getLogger(__name__)
@@ -43,7 +41,7 @@ def _check_numpy() -> bool:
         "langchain will use a pure Python implementation for embedding distance "
         "operations, which may significantly impact performance, especially for large "
         "datasets. For optimal speed and efficiency, consider installing NumPy: "
-        "pip install numpy"
+        "pip install numpy",
     )
     return False
 
@@ -63,11 +61,12 @@ def _embedding_factory() -> Embeddings:
             from langchain_community.embeddings.openai import (  # type: ignore[no-redef]
                 OpenAIEmbeddings,
             )
-        except ImportError:
-            raise ImportError(
+        except ImportError as e:
+            msg = (
                 "Could not import OpenAIEmbeddings. Please install the "
                 "OpenAIEmbeddings package using `pip install langchain-openai`."
             )
+            raise ImportError(msg) from e
     return OpenAIEmbeddings()
 
 
@@ -130,21 +129,23 @@ class _EmbeddingDistanceChainMixin(Chain):
             pass
 
         if not types_:
-            raise ImportError(
+            msg = (
                 "Could not import OpenAIEmbeddings. Please install the "
                 "OpenAIEmbeddings package using `pip install langchain-openai`."
             )
+            raise ImportError(msg)
 
         if isinstance(embeddings, tuple(types_)):
             try:
                 import tiktoken  # noqa: F401
-            except ImportError:
-                raise ImportError(
+            except ImportError as e:
+                msg = (
                     "The tiktoken library is required to use the default "
                     "OpenAI embeddings with embedding distance evaluators."
                     " Please either manually select a different Embeddings object"
                     " or install tiktoken using `pip install tiktoken`."
                 )
+                raise ImportError(msg) from e
         return values
 
     model_config = ConfigDict(
@@ -184,8 +185,8 @@ class _EmbeddingDistanceChainMixin(Chain):
         }
         if metric in metrics:
             return metrics[metric]
-        else:
-            raise ValueError(f"Invalid metric: {metric}")
+        msg = f"Invalid metric: {metric}"
+        raise ValueError(msg)
 
     @staticmethod
     def _cosine_distance(a: Any, b: Any) -> Any:
@@ -200,12 +201,13 @@ class _EmbeddingDistanceChainMixin(Chain):
         """
         try:
             from langchain_community.utils.math import cosine_similarity
-        except ImportError:
-            raise ImportError(
+        except ImportError as e:
+            msg = (
                 "The cosine_similarity function is required to compute cosine distance."
                 " Please install the langchain-community package using"
                 " `pip install langchain-community`."
             )
+            raise ImportError(msg) from e
         return 1.0 - cosine_similarity(a, b)
 
     @staticmethod
@@ -343,7 +345,7 @@ class EmbeddingDistanceEvalChain(_EmbeddingDistanceChainMixin, StringEvaluator):
             Dict[str, Any]: The computed score.
         """
         vectors = self.embeddings.embed_documents(
-            [inputs["prediction"], inputs["reference"]]
+            [inputs["prediction"], inputs["reference"]],
         )
         if _check_numpy():
             np = _import_numpy()
@@ -370,7 +372,7 @@ class EmbeddingDistanceEvalChain(_EmbeddingDistanceChainMixin, StringEvaluator):
             [
                 inputs["prediction"],
                 inputs["reference"],
-            ]
+            ],
         )
         if _check_numpy():
             np = _import_numpy()
@@ -448,7 +450,8 @@ class EmbeddingDistanceEvalChain(_EmbeddingDistanceChainMixin, StringEvaluator):
 
 
 class PairwiseEmbeddingDistanceEvalChain(
-    _EmbeddingDistanceChainMixin, PairwiseStringEvaluator
+    _EmbeddingDistanceChainMixin,
+    PairwiseStringEvaluator,
 ):
     """Use embedding distances to score semantic difference between two predictions.
 
@@ -491,7 +494,7 @@ class PairwiseEmbeddingDistanceEvalChain(
             [
                 inputs["prediction"],
                 inputs["prediction_b"],
-            ]
+            ],
         )
         if _check_numpy():
             np = _import_numpy()
@@ -518,7 +521,7 @@ class PairwiseEmbeddingDistanceEvalChain(
             [
                 inputs["prediction"],
                 inputs["prediction_b"],
-            ]
+            ],
         )
         if _check_numpy():
             np = _import_numpy()

langchain/evaluation/loading.py

@@ -61,18 +61,20 @@ def load_dataset(uri: str) -> list[dict]:
     """
     try:
         from datasets import load_dataset
-    except ImportError:
-        raise ImportError(
+    except ImportError as e:
+        msg = (
             "load_dataset requires the `datasets` package."
             " Please install with `pip install datasets`"
         )
+        raise ImportError(msg) from e
 
     dataset = load_dataset(f"LangChainDatasets/{uri}")
-    return [d for d in dataset["train"]]
+    return list(dataset["train"])
 
 
 _EVALUATOR_MAP: dict[
-    EvaluatorType, Union[type[LLMEvalChain], type[Chain], type[StringEvaluator]]
+    EvaluatorType,
+    Union[type[LLMEvalChain], type[Chain], type[StringEvaluator]],
 ] = {
     EvaluatorType.QA: QAEvalChain,
     EvaluatorType.COT_QA: CotQAEvalChain,
@@ -125,10 +127,11 @@ def load_evaluator(
     >>> evaluator = load_evaluator(EvaluatorType.QA)
     """
     if evaluator not in _EVALUATOR_MAP:
-        raise ValueError(
+        msg = (
             f"Unknown evaluator type: {evaluator}"
             f"\nValid types are: {list(_EVALUATOR_MAP.keys())}"
         )
+        raise ValueError(msg)
     evaluator_cls = _EVALUATOR_MAP[evaluator]
     if issubclass(evaluator_cls, LLMEvalChain):
         try:
@@ -139,27 +142,28 @@ def load_evaluator(
                     from langchain_community.chat_models.openai import (  # type: ignore[no-redef]
                         ChatOpenAI,
                     )
-                except ImportError:
-                    raise ImportError(
+                except ImportError as e:
+                    msg = (
                         "Could not import langchain_openai or fallback onto "
                         "langchain_community. Please install langchain_openai "
                         "or specify a language model explicitly. "
                         "It's recommended to install langchain_openai AND "
                         "specify a language model explicitly."
                     )
+                    raise ImportError(msg) from e
 
             llm = llm or ChatOpenAI(model="gpt-4", seed=42, temperature=0)
         except Exception as e:
-            raise ValueError(
+            msg = (
                 f"Evaluation with the {evaluator_cls} requires a "
                 "language model to function."
                 " Failed to create the default 'gpt-4' model."
                 " Please manually provide an evaluation LLM"
                 " or check your openai credentials."
-            ) from e
+            )
+            raise ValueError(msg) from e
         return evaluator_cls.from_llm(llm=llm, **kwargs)
-    else:
-        return evaluator_cls(**kwargs)
+    return evaluator_cls(**kwargs)
 
 
 def load_evaluators(

langchain/evaluation/parsing/base.py

@@ -5,6 +5,7 @@ from operator import eq
 from typing import Any, Callable, Optional, Union, cast
 
 from langchain_core.utils.json import parse_json_markdown
+from typing_extensions import override
 
 from langchain.evaluation.schema import StringEvaluator
 
@@ -49,6 +50,7 @@ class JsonValidityEvaluator(StringEvaluator):
     def evaluation_name(self) -> str:
         return "json_validity"
 
+    @override
     def _evaluate_strings(
         self,
         prediction: str,
@@ -72,9 +74,9 @@ class JsonValidityEvaluator(StringEvaluator):
         """
         try:
             parse_json_markdown(prediction, parser=json.loads)
-            return {"score": 1}
         except Exception as e:
             return {"score": 0, "reasoning": str(e)}
+        return {"score": 1}
 
 
 class JsonEqualityEvaluator(StringEvaluator):
@@ -132,6 +134,7 @@ class JsonEqualityEvaluator(StringEvaluator):
             return parse_json_markdown(string)
         return string
 
+    @override
     def _evaluate_strings(
         self,
         prediction: str,

langchain/evaluation/parsing/json_distance.py

@@ -2,6 +2,7 @@ import json
 from typing import Any, Callable, Optional, Union
 
 from langchain_core.utils.json import parse_json_markdown
+from typing_extensions import override
 
 from langchain.evaluation.schema import StringEvaluator
 
@@ -47,13 +48,14 @@ class JsonEditDistanceEvaluator(StringEvaluator):
         else:
             try:
                 from rapidfuzz import distance as rfd
-            except ImportError:
-                raise ImportError(
+            except ImportError as e:
+                msg = (
                     "The default string_distance operator for the "
                     " JsonEditDistanceEvaluator requires installation of "
                     "the rapidfuzz package. "
                     "Please install it with `pip install rapidfuzz`."
                 )
+                raise ImportError(msg) from e
             self._string_distance = rfd.DamerauLevenshtein.normalized_distance
         if canonicalize is not None:
             self._canonicalize = canonicalize
@@ -81,6 +83,7 @@ class JsonEditDistanceEvaluator(StringEvaluator):
             return parse_json_markdown(node)
         return node
 
+    @override
     def _evaluate_strings(
         self,
         prediction: str,

langchain/evaluation/parsing/json_schema.py

@@ -1,6 +1,7 @@
 from typing import Any, Union
 
 from langchain_core.utils.json import parse_json_markdown
+from typing_extensions import override
 
 from langchain.evaluation.schema import StringEvaluator
 
@@ -44,11 +45,12 @@ class JsonSchemaEvaluator(StringEvaluator):
         super().__init__()
         try:
             import jsonschema  # noqa: F401
-        except ImportError:
-            raise ImportError(
+        except ImportError as e:
+            msg = (
                 "The JsonSchemaEvaluator requires the jsonschema package."
                 " Please install it with `pip install jsonschema`."
             )
+            raise ImportError(msg) from e
 
     @property
     def requires_input(self) -> bool:
@@ -68,9 +70,12 @@ class JsonSchemaEvaluator(StringEvaluator):
     def _parse_json(self, node: Any) -> Union[dict, list, None, float, bool, int, str]:
         if isinstance(node, str):
             return parse_json_markdown(node)
-        elif hasattr(node, "schema") and callable(getattr(node, "schema")):
-            # Pydantic model
-            return getattr(node, "schema")()
+        if hasattr(node, "model_json_schema") and callable(node.model_json_schema):
+            # Pydantic v2 model
+            return node.model_json_schema()
+        if hasattr(node, "schema") and callable(node.schema):
+            # Pydantic v1 model
+            return node.schema()
         return node
 
     def _validate(self, prediction: Any, schema: Any) -> dict:
@@ -78,12 +83,11 @@ class JsonSchemaEvaluator(StringEvaluator):
 
         try:
             validate(instance=prediction, schema=schema)
-            return {
-                "score": True,
-            }
         except ValidationError as e:
             return {"score": False, "reasoning": repr(e)}
+        return {"score": True}
 
+    @override
     def _evaluate_strings(
         self,
         prediction: Union[str, Any],

langchain/evaluation/qa/__init__.py

@@ -7,4 +7,4 @@ from langchain.evaluation.qa.eval_chain import (
 )
 from langchain.evaluation.qa.generate_chain import QAGenerateChain
 
-__all__ = ["QAEvalChain", "QAGenerateChain", "ContextQAEvalChain", "CotQAEvalChain"]
+__all__ = ["ContextQAEvalChain", "CotQAEvalChain", "QAEvalChain", "QAGenerateChain"]