langchain 0.3.27__py3-none-any.whl → 0.4.0.dev0__py3-none-any.whl

langchain/smith/evaluation/runner_utils.py

@@ -155,9 +155,24 @@ class EvalError(dict):
     """Your architecture raised an error."""
 
     def __init__(self, Error: BaseException, **kwargs: Any) -> None:
+        """Initialize the EvalError with an error and additional attributes.
+
+        Args:
+            Error: The error that occurred.
+            **kwargs: Additional attributes to include in the error.
+        """
         super().__init__(Error=Error, **kwargs)
 
     def __getattr__(self, name: str) -> Any:
+        """Get an attribute from the EvalError.
+
+        Args:
+            name: The name of the attribute to get.
+        Returns:
+            The value of the attribute.
+        Raises:
+            AttributeError: If the attribute does not exist.
+        """
         try:
             return self[name]
         except KeyError as e:
@@ -199,24 +214,24 @@ def _wrap_in_chain_factory(
         return lambda: lcf
     if callable(llm_or_chain_factory):
         if is_traceable_function(llm_or_chain_factory):
-            runnable_ = as_runnable(cast(Callable, llm_or_chain_factory))
+            runnable_ = as_runnable(cast("Callable", llm_or_chain_factory))
             return lambda: runnable_
         try:
             _model = llm_or_chain_factory()  # type: ignore[call-arg]
         except TypeError:
             # It's an arbitrary function, wrap it in a RunnableLambda
-            user_func = cast(Callable, llm_or_chain_factory)
+            user_func = cast("Callable", llm_or_chain_factory)
             sig = inspect.signature(user_func)
             logger.info("Wrapping function %s as RunnableLambda.", sig)
             wrapped = RunnableLambda(user_func)
             return lambda: wrapped
-        constructor = cast(Callable, llm_or_chain_factory)
+        constructor = cast("Callable", llm_or_chain_factory)
         if isinstance(_model, BaseLanguageModel):
             # It's not uncommon to do an LLM constructor instead of raw LLM,
             # so we'll unpack it for the user.
             return _model
-        if is_traceable_function(cast(Callable, _model)):
-            runnable_ = as_runnable(cast(Callable, _model))
+        if is_traceable_function(cast("Callable", _model)):
+            runnable_ = as_runnable(cast("Callable", _model))
             return lambda: runnable_
         if not isinstance(_model, Runnable):
             # This is unlikely to happen - a constructor for a model function
@@ -1089,7 +1104,7 @@ class _DatasetRunContainer:
     ) -> dict:
         results: dict = {}
         for example, output in zip(self.examples, batch_results):
-            row_result = cast(_RowResult, all_eval_results.get(str(example.id), {}))
+            row_result = cast("_RowResult", all_eval_results.get(str(example.id), {}))
             results[str(example.id)] = {
                 "input": example.inputs,
                 "feedback": row_result.get("feedback", []),
@@ -1116,7 +1131,7 @@ class _DatasetRunContainer:
                     result = evaluator(runs_list, self.examples)
                     if isinstance(result, EvaluationResult):
                         result = result.dict()
-                    aggregate_feedback.append(cast(dict, result))
+                    aggregate_feedback.append(cast("dict", result))
                     executor.submit(
                         self.client.create_feedback,
                         **result,
@@ -1133,7 +1148,7 @@ class _DatasetRunContainer:
         all_eval_results: dict = {}
         all_runs: dict = {}
         for c in self.configs:
-            for callback in cast(list, c["callbacks"]):
+            for callback in cast("list", c["callbacks"]):
                 if isinstance(callback, EvaluatorCallbackHandler):
                     eval_results = callback.logged_eval_results
                     for (_, example_id), v in eval_results.items():
@@ -1156,7 +1171,7 @@ class _DatasetRunContainer:
                         },
                     )
                     all_runs[str(callback.example_id)] = run
-        return cast(dict[str, _RowResult], all_eval_results), all_runs
+        return cast("dict[str, _RowResult]", all_eval_results), all_runs
 
     def _collect_test_results(
         self,
@@ -1330,6 +1345,114 @@ async def arun_on_dataset(
     revision_id: Optional[str] = None,
     **kwargs: Any,
 ) -> dict[str, Any]:
+    """Run on dataset.
+
+    Run the Chain or language model on a dataset and store traces
+    to the specified project name.
+
+    For the (usually faster) async version of this function,
+    see :func:`arun_on_dataset`.
+
+    Args:
+        dataset_name: Name of the dataset to run the chain on.
+        llm_or_chain_factory: Language model or Chain constructor to run
+            over the dataset. The Chain constructor is used to permit
+            independent calls on each example without carrying over state.
+        evaluation: Configuration for evaluators to run on the
+            results of the chain
+        concurrency_level: The number of async tasks to run concurrently.
+        project_name: Name of the project to store the traces in.
+            Defaults to {dataset_name}-{chain class name}-{datetime}.
+        project_metadata: Optional metadata to add to the project.
+            Useful for storing information the test variant.
+            (prompt version, model version, etc.)
+        client: LangSmith client to use to access the dataset and to
+            log feedback and run traces.
+        verbose: Whether to print progress.
+        tags: Tags to add to each run in the project.
+        revision_id: Optional revision identifier to assign this test run to
+            track the performance of different versions of your system.
+    Returns:
+        A dictionary containing the run's project name and the resulting model outputs.
+
+    Examples:
+
+    .. code-block:: python
+
+        from langsmith import Client
+        from langchain_openai import ChatOpenAI
+        from langchain.chains import LLMChain
+        from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset
+
+        # Chains may have memory. Passing in a constructor function lets the
+        # evaluation framework avoid cross-contamination between runs.
+        def construct_chain():
+            llm = ChatOpenAI(temperature=0)
+            chain = LLMChain.from_string(
+                llm,
+                "What's the answer to {your_input_key}"
+            )
+            return chain
+
+        # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)
+        evaluation_config = smith_eval.RunEvalConfig(
+            evaluators=[
+                "qa",  # "Correctness" against a reference answer
+                "embedding_distance",
+                smith_eval.RunEvalConfig.Criteria("helpfulness"),
+                smith_eval.RunEvalConfig.Criteria({
+                    "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?"
+                }),
+            ]
+        )
+
+        client = Client()
+        await arun_on_dataset(
+            client,
+            dataset_name="<my_dataset_name>",
+            llm_or_chain_factory=construct_chain,
+            evaluation=evaluation_config,
+        )
+
+    You can also create custom evaluators by subclassing the
+    :class:`StringEvaluator <langchain.evaluation.schema.StringEvaluator>`
+    or LangSmith's `RunEvaluator` classes.
+
+    .. code-block:: python
+
+        from typing import Optional
+        from langchain.evaluation import StringEvaluator
+
+        class MyStringEvaluator(StringEvaluator):
+
+            @property
+            def requires_input(self) -> bool:
+                return False
+
+            @property
+            def requires_reference(self) -> bool:
+                return True
+
+            @property
+            def evaluation_name(self) -> str:
+                return "exact_match"
+
+            def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict:
+                return {"score": prediction == reference}
+
+
+        evaluation_config = smith_eval.RunEvalConfig(
+            custom_evaluators = [MyStringEvaluator()],
+        )
+
+        await arun_on_dataset(
+            client,
+            dataset_name="<my_dataset_name>",
+            llm_or_chain_factory=construct_chain,
+            evaluation=evaluation_config,
+        )
+
+    """  # noqa: E501
     input_mapper = kwargs.pop("input_mapper", None)
     if input_mapper:
         warn_deprecated("0.0.305", message=_INPUT_MAPPER_DEP_WARNING, pending=True)
@@ -1395,6 +1518,114 @@ def run_on_dataset(
     revision_id: Optional[str] = None,
     **kwargs: Any,
 ) -> dict[str, Any]:
+    """Run on dataset.
+
+    Run the Chain or language model on a dataset and store traces
+    to the specified project name.
+
+    For the (usually faster) async version of this function,
+    see :func:`arun_on_dataset`.
+
+    Args:
+        dataset_name: Name of the dataset to run the chain on.
+        llm_or_chain_factory: Language model or Chain constructor to run
+            over the dataset. The Chain constructor is used to permit
+            independent calls on each example without carrying over state.
+        evaluation: Configuration for evaluators to run on the
+            results of the chain
+        concurrency_level: The number of async tasks to run concurrently.
+        project_name: Name of the project to store the traces in.
+            Defaults to {dataset_name}-{chain class name}-{datetime}.
+        project_metadata: Optional metadata to add to the project.
+            Useful for storing information the test variant.
+            (prompt version, model version, etc.)
+        client: LangSmith client to use to access the dataset and to
+            log feedback and run traces.
+        verbose: Whether to print progress.
+        tags: Tags to add to each run in the project.
+        revision_id: Optional revision identifier to assign this test run to
+            track the performance of different versions of your system.
+    Returns:
+        A dictionary containing the run's project name and the resulting model outputs.
+
+    Examples:
+
+    .. code-block:: python
+
+        from langsmith import Client
+        from langchain_openai import ChatOpenAI
+        from langchain.chains import LLMChain
+        from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset
+
+        # Chains may have memory. Passing in a constructor function lets the
+        # evaluation framework avoid cross-contamination between runs.
+        def construct_chain():
+            llm = ChatOpenAI(temperature=0)
+            chain = LLMChain.from_string(
+                llm,
+                "What's the answer to {your_input_key}"
+            )
+            return chain
+
+        # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)
+        evaluation_config = smith_eval.RunEvalConfig(
+            evaluators=[
+                "qa",  # "Correctness" against a reference answer
+                "embedding_distance",
+                smith_eval.RunEvalConfig.Criteria("helpfulness"),
+                smith_eval.RunEvalConfig.Criteria({
+                    "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?"
+                }),
+            ]
+        )
+
+        client = Client()
+        run_on_dataset(
+            client,
+            dataset_name="<my_dataset_name>",
+            llm_or_chain_factory=construct_chain,
+            evaluation=evaluation_config,
+        )
+
+    You can also create custom evaluators by subclassing the
+    :class:`StringEvaluator <langchain.evaluation.schema.StringEvaluator>`
+    or LangSmith's `RunEvaluator` classes.
+
+    .. code-block:: python
+
+        from typing import Optional
+        from langchain.evaluation import StringEvaluator
+
+        class MyStringEvaluator(StringEvaluator):
+
+            @property
+            def requires_input(self) -> bool:
+                return False
+
+            @property
+            def requires_reference(self) -> bool:
+                return True
+
+            @property
+            def evaluation_name(self) -> str:
+                return "exact_match"
+
+            def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict:
+                return {"score": prediction == reference}
+
+
+        evaluation_config = smith_eval.RunEvalConfig(
+            custom_evaluators = [MyStringEvaluator()],
+        )
+
+        run_on_dataset(
+            client,
+            dataset_name="<my_dataset_name>",
+            llm_or_chain_factory=construct_chain,
+            evaluation=evaluation_config,
+        )
+
+    """  # noqa: E501
     input_mapper = kwargs.pop("input_mapper", None)
     if input_mapper:
         warn_deprecated("0.0.305", message=_INPUT_MAPPER_DEP_WARNING, pending=True)
@@ -1456,117 +1687,3 @@ def run_on_dataset(
             )
 
     return container.finish(batch_results, verbose=verbose)
-
-
-_RUN_ON_DATASET_DOCSTRING = """
-Run the Chain or language model on a dataset and store traces
-to the specified project name.
-
-Args:
-    dataset_name: Name of the dataset to run the chain on.
-    llm_or_chain_factory: Language model or Chain constructor to run
-        over the dataset. The Chain constructor is used to permit
-        independent calls on each example without carrying over state.
-    evaluation: Configuration for evaluators to run on the
-        results of the chain
-    concurrency_level: The number of async tasks to run concurrently.
-    project_name: Name of the project to store the traces in.
-        Defaults to {dataset_name}-{chain class name}-{datetime}.
-    project_metadata: Optional metadata to add to the project.
-        Useful for storing information the test variant.
-        (prompt version, model version, etc.)
-    client: LangSmith client to use to access the dataset and to
-        log feedback and run traces.
-    verbose: Whether to print progress.
-    tags: Tags to add to each run in the project.
-    revision_id: Optional revision identifier to assign this test run to
-        track the performance of different versions of your system.
-Returns:
-    A dictionary containing the run's project name and the resulting model outputs.
-
-
-For the (usually faster) async version of this function, see :func:`arun_on_dataset`.
-
-Examples
---------
-
-.. code-block:: python
-
-    from langsmith import Client
-    from langchain_openai import ChatOpenAI
-    from langchain.chains import LLMChain
-    from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset
-
-    # Chains may have memory. Passing in a constructor function lets the
-    # evaluation framework avoid cross-contamination between runs.
-    def construct_chain():
-        llm = ChatOpenAI(temperature=0)
-        chain = LLMChain.from_string(
-            llm,
-            "What's the answer to {your_input_key}"
-        )
-        return chain
-
-    # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)
-    evaluation_config = smith_eval.RunEvalConfig(
-        evaluators=[
-            "qa",  # "Correctness" against a reference answer
-            "embedding_distance",
-            smith_eval.RunEvalConfig.Criteria("helpfulness"),
-            smith_eval.RunEvalConfig.Criteria({
-                "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?"
-            }),
-        ]
-    )
-
-    client = Client()
-    run_on_dataset(
-        client,
-        dataset_name="<my_dataset_name>",
-        llm_or_chain_factory=construct_chain,
-        evaluation=evaluation_config,
-    )
-
-You can also create custom evaluators by subclassing the
-:class:`StringEvaluator <langchain.evaluation.schema.StringEvaluator>`
-or LangSmith's `RunEvaluator` classes.
-
-.. code-block:: python
-
-    from typing import Optional
-    from langchain.evaluation import StringEvaluator
-
-    class MyStringEvaluator(StringEvaluator):
-
-        @property
-        def requires_input(self) -> bool:
-            return False
-
-        @property
-        def requires_reference(self) -> bool:
-            return True
-
-        @property
-        def evaluation_name(self) -> str:
-            return "exact_match"
-
-        def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict:
-            return {"score": prediction == reference}
-
-
-    evaluation_config = smith_eval.RunEvalConfig(
-        custom_evaluators = [MyStringEvaluator()],
-    )
-
-    run_on_dataset(
-        client,
-        dataset_name="<my_dataset_name>",
-        llm_or_chain_factory=construct_chain,
-        evaluation=evaluation_config,
-    )
-"""  # noqa: E501
-run_on_dataset.__doc__ = _RUN_ON_DATASET_DOCSTRING
-arun_on_dataset.__doc__ = _RUN_ON_DATASET_DOCSTRING.replace(
-    "run_on_dataset(",
-    "await arun_on_dataset(",
-)

langchain/smith/evaluation/string_run_evaluator.py

@@ -16,6 +16,7 @@ from langchain_core.load.serializable import Serializable
 from langchain_core.messages import BaseMessage, get_buffer_string, messages_from_dict
 from langsmith import EvaluationResult, RunEvaluator
 from langsmith.schemas import DataType, Example, Run
+from typing_extensions import override
 
 from langchain.chains.base import Chain
 from langchain.evaluation.schema import StringEvaluator
@@ -70,6 +71,15 @@ class LLMStringRunMapper(StringRunMapper):
         raise ValueError(msg)
 
     def serialize_inputs(self, inputs: dict) -> str:
+        """Serialize inputs.
+
+        Args:
+            inputs: The inputs from the run, expected to contain prompts or messages.
+        Returns:
+            The serialized input text from the prompts or messages.
+        Raises:
+            ValueError: If neither prompts nor messages are found in the inputs.
+        """
         if "prompts" in inputs:  # Should we even accept this?
             input_ = "\n\n".join(inputs["prompts"])
         elif "prompt" in inputs:
@@ -82,6 +92,18 @@ class LLMStringRunMapper(StringRunMapper):
         return input_
 
     def serialize_outputs(self, outputs: dict) -> str:
+        """Serialize outputs.
+
+        Args:
+            outputs: The outputs from the run, expected to contain generations.
+
+        Returns:
+            The serialized output text from the first generation.
+
+        Raises:
+            ValueError: If no generations are found in the outputs,
+            or if the generations are empty.
+        """
         if not outputs.get("generations"):
             msg = "Cannot evaluate LLM Run without generations."
             raise ValueError(msg)
@@ -185,6 +207,7 @@ class ChainStringRunMapper(StringRunMapper):
 class ToolStringRunMapper(StringRunMapper):
     """Map an input to the tool."""
 
+    @override
     def map(self, run: Run) -> dict[str, str]:
         if not run.outputs:
             msg = f"Run {run.id} has no outputs to evaluate."
@@ -256,10 +279,12 @@ class StringRunEvaluatorChain(Chain, RunEvaluator):
     """The evaluation chain."""
 
     @property
+    @override
     def input_keys(self) -> list[str]:
         return ["run", "example"]
 
     @property
+    @override
     def output_keys(self) -> list[str]:
         return ["feedback"]
 
@@ -330,6 +355,7 @@ class StringRunEvaluatorChain(Chain, RunEvaluator):
             feedback.evaluator_info[RUN_KEY] = output[RUN_KEY]
         return feedback
 
+    @override
     def evaluate_run(
         self,
         run: Run,
@@ -347,6 +373,7 @@ class StringRunEvaluatorChain(Chain, RunEvaluator):
                 # TODO: Add run ID once we can declare it via callbacks
             )
 
+    @override
     async def aevaluate_run(
         self,
         run: Run,

langchain/storage/encoder_backed.py

@@ -46,6 +46,7 @@ class EncoderBackedStore(BaseStore[K, V]):
         store.mset([(1, 3.14), (2, 2.718)])
         values = store.mget([1, 2])  # Retrieves [3.14, 2.718]
         store.mdelete([1, 2])  # Deletes the keys 1 and 2
+
     """
 
     def __init__(

langchain/tools/python/__init__.py

@@ -1,7 +1,7 @@
 from typing import Any
 
 
-def __getattr__(name: str = "") -> Any:
+def __getattr__(_: str = "") -> Any:
     msg = (
         "This tool has been moved to langchain experiment. "
         "This tool has access to a python REPL. "

{langchain-0.3.27.dist-info → langchain-0.4.0.dev0.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: langchain
-Version: 0.3.27
+Version: 0.4.0.dev0
 Summary: Building applications with LLMs through composability
 License: MIT
 Project-URL: Source Code, https://github.com/langchain-ai/langchain/tree/master/libs/langchain
 Project-URL: Release Notes, https://github.com/langchain-ai/langchain/releases?q=tag%3A%22langchain%3D%3D0%22&expanded=true
 Project-URL: repository, https://github.com/langchain-ai/langchain
 Requires-Python: <4.0,>=3.9
-Requires-Dist: langchain-core<1.0.0,>=0.3.72
+Requires-Dist: langchain-core<1.0.0,>=0.4.0.dev0
 Requires-Dist: langchain-text-splitters<1.0.0,>=0.3.9
 Requires-Dist: langsmith>=0.1.17
 Requires-Dist: pydantic<3.0.0,>=2.7.4
@@ -21,12 +21,6 @@ Provides-Extra: anthropic
 Requires-Dist: langchain-anthropic; extra == "anthropic"
 Provides-Extra: openai
 Requires-Dist: langchain-openai; extra == "openai"
-Provides-Extra: azure-ai
-Requires-Dist: langchain-azure-ai; extra == "azure-ai"
-Provides-Extra: cohere
-Requires-Dist: langchain-cohere; extra == "cohere"
-Provides-Extra: google-vertexai
-Requires-Dist: langchain-google-vertexai; extra == "google-vertexai"
 Provides-Extra: google-genai
 Requires-Dist: langchain-google-genai; extra == "google-genai"
 Provides-Extra: fireworks
@@ -41,12 +35,8 @@ Provides-Extra: huggingface
 Requires-Dist: langchain-huggingface; extra == "huggingface"
 Provides-Extra: groq
 Requires-Dist: langchain-groq; extra == "groq"
-Provides-Extra: aws
-Requires-Dist: langchain-aws; extra == "aws"
 Provides-Extra: deepseek
 Requires-Dist: langchain-deepseek; extra == "deepseek"
-Provides-Extra: xai
-Requires-Dist: langchain-xai; extra == "xai"
 Provides-Extra: perplexity
 Requires-Dist: langchain-perplexity; extra == "perplexity"
 Description-Content-Type: text/markdown