langchain 0.3.26__py3-none-any.whl → 0.4.0.dev0__py3-none-any.whl

langchain/evaluation/loading.py

@@ -58,21 +58,24 @@ def load_dataset(uri: str) -> list[dict]:
 
         from langchain.evaluation import load_dataset
         ds = load_dataset("llm-math")
+
     """
     try:
         from datasets import load_dataset
-    except ImportError:
-        raise ImportError(
+    except ImportError as e:
+        msg = (
             "load_dataset requires the `datasets` package."
             " Please install with `pip install datasets`"
         )
+        raise ImportError(msg) from e
 
     dataset = load_dataset(f"LangChainDatasets/{uri}")
-    return [d for d in dataset["train"]]
+    return list(dataset["train"])
 
 
 _EVALUATOR_MAP: dict[
-    EvaluatorType, Union[type[LLMEvalChain], type[Chain], type[StringEvaluator]]
+    EvaluatorType,
+    Union[type[LLMEvalChain], type[Chain], type[StringEvaluator]],
 ] = {
     EvaluatorType.QA: QAEvalChain,
     EvaluatorType.COT_QA: CotQAEvalChain,
@@ -125,10 +128,11 @@ def load_evaluator(
     >>> evaluator = load_evaluator(EvaluatorType.QA)
     """
     if evaluator not in _EVALUATOR_MAP:
-        raise ValueError(
+        msg = (
             f"Unknown evaluator type: {evaluator}"
             f"\nValid types are: {list(_EVALUATOR_MAP.keys())}"
         )
+        raise ValueError(msg)
     evaluator_cls = _EVALUATOR_MAP[evaluator]
     if issubclass(evaluator_cls, LLMEvalChain):
         try:
@@ -139,27 +143,28 @@ def load_evaluator(
                     from langchain_community.chat_models.openai import (  # type: ignore[no-redef]
                         ChatOpenAI,
                     )
-                except ImportError:
-                    raise ImportError(
+                except ImportError as e:
+                    msg = (
                         "Could not import langchain_openai or fallback onto "
                         "langchain_community. Please install langchain_openai "
                         "or specify a language model explicitly. "
                         "It's recommended to install langchain_openai AND "
                         "specify a language model explicitly."
                     )
+                    raise ImportError(msg) from e
 
             llm = llm or ChatOpenAI(model="gpt-4", seed=42, temperature=0)
         except Exception as e:
-            raise ValueError(
+            msg = (
                 f"Evaluation with the {evaluator_cls} requires a "
                 "language model to function."
                 " Failed to create the default 'gpt-4' model."
                 " Please manually provide an evaluation LLM"
                 " or check your openai credentials."
-            ) from e
+            )
+            raise ValueError(msg) from e
         return evaluator_cls.from_llm(llm=llm, **kwargs)
-    else:
-        return evaluator_cls(**kwargs)
+    return evaluator_cls(**kwargs)
 
 
 def load_evaluators(

langchain/evaluation/parsing/base.py

@@ -5,6 +5,7 @@ from operator import eq
 from typing import Any, Callable, Optional, Union, cast
 
 from langchain_core.utils.json import parse_json_markdown
+from typing_extensions import override
 
 from langchain.evaluation.schema import StringEvaluator
 
@@ -34,21 +35,26 @@ class JsonValidityEvaluator(StringEvaluator):
         {'score': 0, 'reasoning': 'Expecting property name enclosed in double quotes'}
     """
 
-    def __init__(self, **kwargs: Any) -> None:
+    def __init__(self, **_: Any) -> None:
+        """Initialize the JsonValidityEvaluator."""
         super().__init__()
 
     @property
+    @override
     def requires_input(self) -> bool:
         return False
 
     @property
+    @override
     def requires_reference(self) -> bool:
         return False
 
     @property
+    @override
     def evaluation_name(self) -> str:
         return "json_validity"
 
+    @override
     def _evaluate_strings(
         self,
         prediction: str,
@@ -72,9 +78,9 @@ class JsonValidityEvaluator(StringEvaluator):
         """
         try:
             parse_json_markdown(prediction, parser=json.loads)
-            return {"score": 1}
         except Exception as e:
             return {"score": 0, "reasoning": str(e)}
+        return {"score": 1}
 
 
 class JsonEqualityEvaluator(StringEvaluator):
@@ -108,19 +114,28 @@ class JsonEqualityEvaluator(StringEvaluator):
 
     """
 
-    def __init__(self, operator: Optional[Callable] = None, **kwargs: Any) -> None:
+    def __init__(self, operator: Optional[Callable] = None, **_: Any) -> None:
+        """Initialize the JsonEqualityEvaluator.
+
+        Args:
+            operator: A custom operator to compare the parsed JSON objects.
+                Defaults to equality (`eq`).
+        """
         super().__init__()
         self.operator = operator or eq
 
     @property
+    @override
     def requires_input(self) -> bool:
         return False
 
     @property
+    @override
     def requires_reference(self) -> bool:
         return True
 
     @property
+    @override
     def evaluation_name(self) -> str:
         return "json_equality"
 
@@ -132,6 +147,7 @@ class JsonEqualityEvaluator(StringEvaluator):
             return parse_json_markdown(string)
         return string
 
+    @override
     def _evaluate_strings(
         self,
         prediction: str,
@@ -150,7 +166,7 @@ class JsonEqualityEvaluator(StringEvaluator):
             dict: A dictionary containing the evaluation score.
         """
         parsed = self._parse_json(prediction)
-        label = self._parse_json(cast(str, reference))
+        label = self._parse_json(cast("str", reference))
         if isinstance(label, list):
             if not isinstance(parsed, list):
                 return {"score": 0}

langchain/evaluation/parsing/json_distance.py

@@ -2,6 +2,7 @@ import json
 from typing import Any, Callable, Optional, Union
 
 from langchain_core.utils.json import parse_json_markdown
+from typing_extensions import override
 
 from langchain.evaluation.schema import StringEvaluator
 
@@ -14,13 +15,6 @@ class JsonEditDistanceEvaluator(StringEvaluator):
     after parsing them and converting them to a canonical format (i.e., whitespace and key order are normalized).
     It can be customized with alternative distance and canonicalization functions.
 
-    Args:
-        string_distance (Optional[Callable[[str, str], float]]): A callable that computes the distance between two strings.
-            If not provided, a Damerau-Levenshtein distance from the `rapidfuzz` package will be used.
-        canonicalize (Optional[Callable[[Any], Any]]): A callable that converts a parsed JSON object into its canonical string form.
-            If not provided, the default behavior is to serialize the JSON with sorted keys and no extra whitespace.
-        **kwargs (Any): Additional keyword arguments.
-
     Attributes:
         _string_distance (Callable[[str, str], float]): The internal distance computation function.
         _canonicalize (Callable[[Any], Any]): The internal canonicalization function.
@@ -39,21 +33,37 @@ class JsonEditDistanceEvaluator(StringEvaluator):
         self,
         string_distance: Optional[Callable[[str, str], float]] = None,
         canonicalize: Optional[Callable[[Any], Any]] = None,
-        **kwargs: Any,
+        **_: Any,
     ) -> None:
+        """Initialize the JsonEditDistanceEvaluator.
+
+        Args:
+            string_distance: A callable that computes the distance between two strings.
+                If not provided, a Damerau-Levenshtein distance from the `rapidfuzz`
+                package will be used.
+            canonicalize: A callable that converts a parsed JSON object into its
+                canonical string form.
+                If not provided, the default behavior is to serialize the JSON with
+                sorted keys and no extra whitespace.
+
+        Raises:
+            ImportError: If the `rapidfuzz` package is not installed and no
+            `string_distance` function is provided.
+        """
         super().__init__()
         if string_distance is not None:
             self._string_distance = string_distance
         else:
             try:
                 from rapidfuzz import distance as rfd
-            except ImportError:
-                raise ImportError(
+            except ImportError as e:
+                msg = (
                     "The default string_distance operator for the "
                     " JsonEditDistanceEvaluator requires installation of "
                     "the rapidfuzz package. "
                     "Please install it with `pip install rapidfuzz`."
                 )
+                raise ImportError(msg) from e
             self._string_distance = rfd.DamerauLevenshtein.normalized_distance
         if canonicalize is not None:
             self._canonicalize = canonicalize
@@ -65,14 +75,17 @@ class JsonEditDistanceEvaluator(StringEvaluator):
             )
 
     @property
+    @override
     def requires_input(self) -> bool:
         return False
 
     @property
+    @override
     def requires_reference(self) -> bool:
         return True
 
     @property
+    @override
     def evaluation_name(self) -> str:
         return "json_edit_distance"
 
@@ -81,6 +94,7 @@ class JsonEditDistanceEvaluator(StringEvaluator):
             return parse_json_markdown(node)
         return node
 
+    @override
     def _evaluate_strings(
         self,
         prediction: str,

langchain/evaluation/parsing/json_schema.py

@@ -1,6 +1,7 @@
 from typing import Any, Union
 
 from langchain_core.utils.json import parse_json_markdown
+from typing_extensions import override
 
 from langchain.evaluation.schema import StringEvaluator
 
@@ -32,23 +33,21 @@ class JsonSchemaEvaluator(StringEvaluator):
 
     """  # noqa: E501
 
-    def __init__(self, **kwargs: Any) -> None:
+    def __init__(self, **_: Any) -> None:
         """Initializes the JsonSchemaEvaluator.
 
-        Args:
-            kwargs: Additional keyword arguments.
-
         Raises:
             ImportError: If the jsonschema package is not installed.
         """
         super().__init__()
         try:
             import jsonschema  # noqa: F401
-        except ImportError:
-            raise ImportError(
+        except ImportError as e:
+            msg = (
                 "The JsonSchemaEvaluator requires the jsonschema package."
                 " Please install it with `pip install jsonschema`."
             )
+            raise ImportError(msg) from e
 
     @property
     def requires_input(self) -> bool:
@@ -68,9 +67,12 @@ class JsonSchemaEvaluator(StringEvaluator):
     def _parse_json(self, node: Any) -> Union[dict, list, None, float, bool, int, str]:
         if isinstance(node, str):
             return parse_json_markdown(node)
-        elif hasattr(node, "schema") and callable(getattr(node, "schema")):
-            # Pydantic model
-            return getattr(node, "schema")()
+        if hasattr(node, "model_json_schema") and callable(node.model_json_schema):
+            # Pydantic v2 model
+            return node.model_json_schema()
+        if hasattr(node, "schema") and callable(node.schema):
+            # Pydantic v1 model
+            return node.schema()
         return node
 
     def _validate(self, prediction: Any, schema: Any) -> dict:
@@ -78,12 +80,11 @@ class JsonSchemaEvaluator(StringEvaluator):
 
         try:
             validate(instance=prediction, schema=schema)
-            return {
-                "score": True,
-            }
         except ValidationError as e:
             return {"score": False, "reasoning": repr(e)}
+        return {"score": True}
 
+    @override
     def _evaluate_strings(
         self,
         prediction: Union[str, Any],

langchain/evaluation/qa/__init__.py

@@ -7,4 +7,4 @@ from langchain.evaluation.qa.eval_chain import (
 )
 from langchain.evaluation.qa.generate_chain import QAGenerateChain
 
-__all__ = ["QAEvalChain", "QAGenerateChain", "ContextQAEvalChain", "CotQAEvalChain"]
+__all__ = ["ContextQAEvalChain", "CotQAEvalChain", "QAEvalChain", "QAGenerateChain"]

langchain/evaluation/qa/eval_chain.py

@@ -11,6 +11,7 @@ from langchain_core.callbacks import Callbacks
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.prompts import PromptTemplate
 from pydantic import ConfigDict
+from typing_extensions import override
 
 from langchain.chains.llm import LLMChain
 from langchain.evaluation.qa.eval_prompt import CONTEXT_PROMPT, COT_PROMPT, PROMPT
@@ -23,7 +24,7 @@ def _get_score(text: str) -> Optional[tuple[str, int]]:
     if match:
         if match.group(1).upper() == "CORRECT":
             return "CORRECT", 1
-        elif match.group(1).upper() == "INCORRECT":
+        if match.group(1).upper() == "INCORRECT":
             return "INCORRECT", 0
     try:
         first_word = (
@@ -31,7 +32,7 @@ def _get_score(text: str) -> Optional[tuple[str, int]]:
         )
         if first_word.upper() == "CORRECT":
             return "CORRECT", 1
-        elif first_word.upper() == "INCORRECT":
+        if first_word.upper() == "INCORRECT":
             return "INCORRECT", 0
         last_word = (
             text.strip()
@@ -40,7 +41,7 @@ def _get_score(text: str) -> Optional[tuple[str, int]]:
         )
         if last_word.upper() == "CORRECT":
             return "CORRECT", 1
-        elif last_word.upper() == "INCORRECT":
+        if last_word.upper() == "INCORRECT":
             return "INCORRECT", 0
     except IndexError:
         pass
@@ -79,18 +80,22 @@ class QAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
     )
 
     @classmethod
+    @override
     def is_lc_serializable(cls) -> bool:
         return False
 
     @property
+    @override
     def evaluation_name(self) -> str:
         return "correctness"
 
     @property
+    @override
     def requires_reference(self) -> bool:
         return True
 
     @property
+    @override
     def requires_input(self) -> bool:
         return True
 
@@ -119,10 +124,11 @@ class QAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
         prompt = prompt or PROMPT
         expected_input_vars = {"query", "answer", "result"}
         if expected_input_vars != set(prompt.input_variables):
-            raise ValueError(
+            msg = (
                 f"Input variables should be {expected_input_vars}, "
                 f"but got {prompt.input_variables}"
             )
+            raise ValueError(msg)
         return cls(llm=llm, prompt=prompt, **kwargs)
 
     def evaluate(
@@ -153,6 +159,7 @@ class QAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
             parsed_result[RUN_KEY] = result[RUN_KEY]
         return parsed_result
 
+    @override
     def _evaluate_strings(
         self,
         *,
@@ -188,6 +195,7 @@ class QAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
         )
         return self._prepare_output(result)
 
+    @override
     async def _aevaluate_strings(
         self,
         *,
@@ -210,6 +218,7 @@ class ContextQAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
     """LLM Chain for evaluating QA w/o GT based on context"""
 
     @classmethod
+    @override
     def is_lc_serializable(cls) -> bool:
         return False
 
@@ -231,12 +240,14 @@ class ContextQAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
     def _validate_input_vars(cls, prompt: PromptTemplate) -> None:
         expected_input_vars = {"query", "context", "result"}
         if expected_input_vars != set(prompt.input_variables):
-            raise ValueError(
+            msg = (
                 f"Input variables should be {expected_input_vars}, "
                 f"but got {prompt.input_variables}"
             )
+            raise ValueError(msg)
 
     @property
+    @override
     def evaluation_name(self) -> str:
         return "Contextual Accuracy"
 
@@ -294,6 +305,7 @@ class ContextQAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
             parsed_result[RUN_KEY] = result[RUN_KEY]
         return parsed_result
 
+    @override
     def _evaluate_strings(
         self,
         *,
@@ -315,6 +327,7 @@ class ContextQAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
         )
         return self._prepare_output(result)
 
+    @override
     async def _aevaluate_strings(
         self,
         *,
@@ -337,10 +350,12 @@ class CotQAEvalChain(ContextQAEvalChain):
     """LLM Chain for evaluating QA using chain of thought reasoning."""
 
     @classmethod
+    @override
     def is_lc_serializable(cls) -> bool:
         return False
 
     @property
+    @override
     def evaluation_name(self) -> str:
         return "COT Contextual Accuracy"
 

langchain/evaluation/qa/eval_prompt.py

@@ -1,4 +1,3 @@
-# flake8: noqa
 from langchain_core.prompts import PromptTemplate
 
 template = """You are a teacher grading a quiz.
@@ -10,12 +9,12 @@ STUDENT ANSWER: student's answer here
 TRUE ANSWER: true answer here
 GRADE: CORRECT or INCORRECT here
 
-Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin!
 
 QUESTION: {query}
 STUDENT ANSWER: {result}
 TRUE ANSWER: {answer}
-GRADE:"""
+GRADE:"""  # noqa: E501
 PROMPT = PromptTemplate(
     input_variables=["query", "result", "answer"], template=template
 )
@@ -29,12 +28,12 @@ CONTEXT: context the question is about here
 STUDENT ANSWER: student's answer here
 GRADE: CORRECT or INCORRECT here
 
-Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin!
 
 QUESTION: {query}
 CONTEXT: {context}
 STUDENT ANSWER: {result}
-GRADE:"""
+GRADE:"""  # noqa: E501
 CONTEXT_PROMPT = PromptTemplate(
     input_variables=["query", "context", "result"], template=context_template
 )
@@ -51,12 +50,12 @@ STUDENT ANSWER: student's answer here
 EXPLANATION: step by step reasoning here
 GRADE: CORRECT or INCORRECT here
 
-Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin!
 
 QUESTION: {query}
 CONTEXT: {context}
 STUDENT ANSWER: {result}
-EXPLANATION:"""
+EXPLANATION:"""  # noqa: E501
 COT_PROMPT = PromptTemplate(
     input_variables=["query", "context", "result"], template=cot_template
 )
@@ -72,7 +71,7 @@ template = """You are comparing a submitted answer to an expert answer on a give
 [Submission]: {result}
 ***
 [END DATA]
-Compare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either "CORRECT" or "INCORRECT" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line."""
+Compare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either "CORRECT" or "INCORRECT" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line."""  # noqa: E501
 
 SQL_PROMPT = PromptTemplate(
     input_variables=["query", "answer", "result"], template=template

langchain/evaluation/qa/generate_chain.py

@@ -7,13 +7,15 @@ from typing import Any
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.output_parsers import BaseLLMOutputParser
 from pydantic import Field
+from typing_extensions import override
 
 from langchain.chains.llm import LLMChain
 from langchain.evaluation.qa.generate_prompt import PROMPT
 from langchain.output_parsers.regex import RegexParser
 
 _QA_OUTPUT_PARSER = RegexParser(
-    regex=r"QUESTION: (.*?)\n+ANSWER: (.*)", output_keys=["query", "answer"]
+    regex=r"QUESTION: (.*?)\n+ANSWER: (.*)",
+    output_keys=["query", "answer"],
 )
 
 
@@ -24,6 +26,7 @@ class QAGenerateChain(LLMChain):
     output_key: str = "qa_pairs"
 
     @classmethod
+    @override
     def is_lc_serializable(cls) -> bool:
         return False
 

langchain/evaluation/qa/generate_prompt.py

@@ -1,8 +1,6 @@
-# flake8: noqa
-from langchain.output_parsers.regex import RegexParser
 from langchain_core.prompts import PromptTemplate
 
-template = """You are a teacher coming up with questions to ask on a quiz. 
+template = """You are a teacher coming up with questions to ask on a quiz.
 Given the following document, please generate a question and answer based on that document.
 
 Example Format:
@@ -16,7 +14,7 @@ These questions should be detailed and be based explicitly on information in the
 
 <Begin Document>
 {doc}
-<End Document>"""
+<End Document>"""  # noqa: E501
 PROMPT = PromptTemplate(
     input_variables=["doc"],
     template=template,

langchain/evaluation/regex_match/base.py

@@ -1,6 +1,8 @@
 import re
 from typing import Any
 
+from typing_extensions import override
+
 from langchain.evaluation.schema import StringEvaluator
 
 
@@ -27,7 +29,12 @@ class RegexMatchStringEvaluator(StringEvaluator):
         )  # This will return {'score': 1.0} as the prediction matches the second pattern in the union
     """  # noqa: E501
 
-    def __init__(self, *, flags: int = 0, **kwargs: Any):  # Default is no flags
+    def __init__(self, *, flags: int = 0, **_: Any):  # Default is no flags
+        """Initialize the RegexMatchStringEvaluator.
+
+        Args:
+            flags: Flags to use for the regex match. Defaults to 0 (no flags).
+        """
         super().__init__()
         self.flags = flags
 
@@ -65,6 +72,7 @@ class RegexMatchStringEvaluator(StringEvaluator):
         """
         return "regex_match"
 
+    @override
     def _evaluate_strings(  # type: ignore[override]
         self,
         *,