trulens-core 1.0.1__tar.gz

trulens_core-1.0.1/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.1
+Name: trulens-core
+Version: 1.0.1
+Summary: Library to systematically track and evaluate LLM based applications.
+Home-page: https://trulens.org/
+License: MIT
+Author: Snowflake Inc.
+Author-email: ml-observability-wg-dl@snowflake.com
+Requires-Python: >=3.9,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: alembic (>=1.8.1,<2.0.0)
+Requires-Dist: dill (>=0.3.8,<0.4.0)
+Requires-Dist: munch (>=2.5,<3.0)
+Requires-Dist: nest-asyncio (>=1.5,<2.0)
+Requires-Dist: numpy (>=1.23)
+Requires-Dist: pandas (>=2.1,<3.0)
+Requires-Dist: pydantic (>=2,<3)
+Requires-Dist: python-dotenv (>=0.21,<2.0)
+Requires-Dist: rich (>=13.6,<14.0)
+Requires-Dist: sqlalchemy (>=2.0,<3.0)
+Requires-Dist: tqdm (>=4.2.0)
+Requires-Dist: typing_extensions (>=4.9,<5.0)
+Project-URL: Documentation, https://trulens.org/trulens/getting_started/
+Project-URL: Repository, https://github.com/truera/trulens
+Description-Content-Type: text/markdown
+
+# trulens-core
+

trulens_core-1.0.1/README.md

@@ -0,0 +1 @@
+# trulens-core

trulens_core-1.0.1/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = [
+  "poetry-core",
+]
+
+[tool.poetry]
+name = "trulens-core"
+version = "1.0.1"
+description = "Library to systematically track and evaluate LLM based applications."
+authors = [
+  "Snowflake Inc. <ml-observability-wg-dl@snowflake.com>",
+]
+license = "MIT"
+readme = "README.md"
+packages = [
+  { include = "trulens" },
+]
+homepage = "https://trulens.org/"
+documentation = "https://trulens.org/trulens/getting_started/"
+repository = "https://github.com/truera/trulens"
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+numpy = ">=1.23"
+munch = "^2.5"
+dill = "^0.3.8"
+typing_extensions = "^4.9"
+pydantic = "^2"
+pandas = "^2.1"
+rich = "^13.6"
+sqlalchemy = "^2.0"
+alembic = "^1.8.1"
+nest-asyncio = "^1.5"
+python-dotenv = ">=0.21,<2.0"
+tqdm = { version = ">=4.2.0", optional = true }
+
+[tool.poetry.group.openai]
+optional = true
+
+[tool.poetry.group.openai.dependencies]
+openai = "^1.37"
+httpx = "^0.27"

trulens_core-1.0.1/trulens/apps/basic.py

@@ -0,0 +1,169 @@
+"""
+# Basic input output instrumentation and monitoring.
+"""
+
+from inspect import BoundArguments
+from inspect import Signature
+from inspect import signature
+import logging
+from pprint import PrettyPrinter
+from typing import Any, Callable, ClassVar, Dict, Optional
+
+from pydantic import Field
+from trulens.core.app import App
+from trulens.core.instruments import ClassFilter
+from trulens.core.instruments import Instrument
+from trulens.core.utils.pyschema import Class
+from trulens.core.utils.pyschema import FunctionOrMethod
+
+logger = logging.getLogger(__name__)
+
+pp = PrettyPrinter()
+
+
+class TruWrapperApp:
+    """Wrapper of basic apps.
+
+    This will be wrapped by instrumentation.
+
+    Warning:
+        Because `TruWrapperApp` may wrap different types of callables, we cannot
+        patch the signature to anything consistent. Because of this, the
+        dashboard/record for this call will have `*args`, `**kwargs` instead of
+        what the app actually uses. We also need to adjust the main_input lookup
+        to get the correct signature. See note there.
+    """
+
+    def _call(self, *args, **kwargs):
+        return self._call_fn(*args, **kwargs)
+
+    def __call__(self, *args, **kwargs):
+        return self._call(*args, **kwargs)
+
+    def __init__(self, call_fn: Callable):
+        self._call_fn = call_fn
+
+
+class TruBasicCallableInstrument(Instrument):
+    """Basic app instrumentation."""
+
+    class Default:
+        """Default instrumentation specification for basic apps."""
+
+        CLASSES = lambda: {TruWrapperApp}
+
+        # Instrument only methods with these names and of these classes.
+        METHODS: Dict[str, ClassFilter] = {"_call": TruWrapperApp}
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            include_classes=TruBasicCallableInstrument.Default.CLASSES(),
+            include_methods=TruBasicCallableInstrument.Default.METHODS,
+            *args,
+            **kwargs,
+        )
+
+
+class TruBasicApp(App):
+    """Instantiates a Basic app that makes little assumptions.
+
+    Assumes input text and output text.
+
+    Example:
+        ```python
+        def custom_application(prompt: str) -> str:
+            return "a response"
+
+        from trulens.apps.basic import TruBasicApp
+        # f_lang_match, f_qa_relevance, f_context_relevance are feedback functions
+        tru_recorder = TruBasicApp(custom_application,
+            app_name="Custom Application",
+            app_version="1",
+            feedbacks=[f_lang_match, f_qa_relevance, f_context_relevance])
+
+        # Basic app works by turning your callable into an app
+        # This app is accessible with the `app` attribute in the recorder
+        with tru_recorder as recording:
+            tru_recorder.app(question)
+
+        tru_record = recording.records[0]
+        ```
+
+        See [Feedback
+        Functions](https://www.trulens.org/trulens/api/feedback/) for
+        instantiating feedback functions.
+
+    Args:
+        text_to_text: A str to str callable.
+
+        app: A TruWrapperApp instance. If not provided, `text_to_text` must
+            be provided.
+
+        **kwargs: Additional arguments to pass to [App][trulens.core.app.App]
+            and [AppDefinition][trulens.core.schema.app.AppDefinition]
+    """
+
+    model_config: ClassVar[dict] = dict(arbitrary_types_allowed=True)
+
+    app: TruWrapperApp
+    """The app to be instrumented."""
+
+    root_callable: ClassVar[FunctionOrMethod] = Field(
+        default_factory=lambda: FunctionOrMethod.of_callable(
+            TruWrapperApp._call
+        )
+    )
+    """The root callable to be instrumented.
+
+    This is the method that will be called by the main_input method."""
+
+    def __init__(
+        self,
+        text_to_text: Optional[Callable[[str], str]] = None,
+        app: Optional[TruWrapperApp] = None,
+        **kwargs: Any,
+    ):
+        if text_to_text is not None:
+            app = TruWrapperApp(text_to_text)
+        else:
+            assert (
+                app is not None
+            ), "Need to provide either `app: TruWrapperApp` or a `text_to_text: Callable`."
+
+        kwargs["app"] = app
+        kwargs["root_class"] = Class.of_object(app)
+        kwargs["instrument"] = TruBasicCallableInstrument(app=self)
+
+        super().__init__(**kwargs)
+
+    def main_call(self, human: str) -> str:
+        # If available, a single text to a single text invocation of this app.
+
+        return self.app._call(human)
+
+    def main_input(
+        self, func: Callable, sig: Signature, bindings: BoundArguments
+    ) -> str:
+        if func == getattr(TruWrapperApp._call, Instrument.INSTRUMENT):
+            # If func is the wrapper app _call, replace the signature and
+            # bindings based on the actual containing callable instead of
+            # self.app._call . This needs to be done since the a TruWrapperApp
+            # may be wrapping apps with different signatures on their callables
+            # so TruWrapperApp._call cannot have a consistent signature
+            # statically. Note also we are looking up the Instrument.INSTRUMENT
+            # attribute here since the method is instrumented and overridden by
+            # another wrapper in the process with the original accessible at
+            # this attribute.
+
+            sig = signature(self.app._call_fn)
+            # Skipping self as TruWrapperApp._call takes in self, but
+            # self.app._call_fn does not.
+            bindings = sig.bind(*bindings.args[1:], **bindings.kwargs)
+
+        return super().main_input(func, sig, bindings)
+
+    def call_with_record(self, *args, **kwargs) -> None:
+        self._throw_dep_message(method="call", is_async=False, with_record=True)
+
+
+TruBasicApp.model_rebuild()