trulens-eval 0.33.0__tar.gz

trulens_eval-0.33.0/MANIFEST.in

@@ -0,0 +1,6 @@
+include trulens_eval/LICENSE
+include trulens_eval/requirements.txt
+include trulens_eval/requirements.optional.txt
+include trulens_eval/ux/trulens_logo.svg
+include trulens_eval/database/migrations/alembic.ini
+recursive-include trulens_eval/react_components/record_viewer/dist  *

trulens_eval-0.33.0/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.1
+Name: trulens_eval
+Version: 0.33.0
+Summary: Library to systematically track and evaluate LLM based applications.
+Home-page: https://www.trulens.org
+Author: Truera Inc
+Author-email: all@truera.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >= 3.8, < 3.13
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.23.5
+Requires-Dist: frozendict>=2.3.8
+Requires-Dist: munch>=3.0.0
+Requires-Dist: dill>=0.3.7
+Requires-Dist: tqdm>=4.66.1
+Requires-Dist: nltk>=3.8.1
+Requires-Dist: requests>=2.31.0
+Requires-Dist: nest-asyncio>=1.5.8
+Requires-Dist: typing_extensions>=4.9.0
+Requires-Dist: psutil>=5.9.8
+Requires-Dist: pip>=24.0
+Requires-Dist: packaging>=23.2
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: merkle-json>=1.0.0
+Requires-Dist: langchain>=0.1.14
+Requires-Dist: langchain-core>=0.1.6
+Requires-Dist: langchain-community>=0.0.38
+Requires-Dist: typing-inspect>=0.8.0
+Requires-Dist: millify>=0.1.1
+Requires-Dist: humanize>=4.6.0
+Requires-Dist: streamlit>=1.33.0
+Requires-Dist: streamlit-aggrid==0.3.4
+Requires-Dist: streamlit-extras>=0.4.0
+Requires-Dist: streamlit-pills>=0.3.0
+Requires-Dist: rich>=13.6.0
+Requires-Dist: sqlalchemy>=2.0.19
+Requires-Dist: alembic>=1.11.2
+
+<!---
+start of docs/trulens_eval/intro.md
+NOTE: This content is from docs/trulens_eval/intro.md and is merged into
+trulens_eval/README.md . If you are editing README.md, your changes will be overwritten.
+-->
+# Welcome to TruLens-Eval!
+
+![TruLens](https://www.trulens.org/assets/images/Neural_Network_Explainability.png)
+
+**Don't just vibe-check your llm app!** Systematically evaluate and track your
+LLM experiments with TruLens. As you develop your app including prompts, models,
+retreivers, knowledge sources and more, *TruLens-Eval* is the tool you need to
+understand its performance.
+
+Fine-grained, stack-agnostic instrumentation and comprehensive evaluations help
+you to identify failure modes & systematically iterate to improve your
+application.
+
+Read more about the core concepts behind TruLens including [Feedback Functions](https://www.trulens.org/trulens_eval/getting_started/core_concepts/feedback_functions/), 
+[The RAG Triad](https://www.trulens.org/trulens_eval/getting_started/core_concepts/rag_triad/),
+and [Honest, Harmless and Helpful Evals](https://www.trulens.org/trulens_eval/getting_started/core_concepts/honest_harmless_helpful_evals/).
+
+## TruLens in the development workflow
+
+Build your first prototype then connect instrumentation and logging with
+TruLens. Decide what feedbacks you need, and specify them with TruLens to run
+alongside your app. Then iterate and compare versions of your app in an
+easy-to-use user interface 👇
+
+![Architecture
+Diagram](https://www.trulens.org/assets/images/TruLens_Architecture.png)
+
+## Installation and Setup
+
+Install the trulens-eval pip package from PyPI.
+
+```bash
+    pip install trulens-eval
+```
+
+## Quick Usage
+
+Walk through how to instrument and evaluate a RAG built from scratch with
+TruLens.
+
+[![Open In
+Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/quickstart.ipynb)
+
+### 💡 Contributing
+
+Interested in contributing? See our [contributing
+guide](https://www.trulens.org/trulens_eval/contributing/) for more details.
+<!---
+end of docs/trulens_eval/intro.md
+-->

trulens_eval-0.33.0/README.md

@@ -0,0 +1,55 @@
+<!---
+start of docs/trulens_eval/intro.md
+NOTE: This content is from docs/trulens_eval/intro.md and is merged into
+trulens_eval/README.md . If you are editing README.md, your changes will be overwritten.
+-->
+# Welcome to TruLens-Eval!
+
+![TruLens](https://www.trulens.org/assets/images/Neural_Network_Explainability.png)
+
+**Don't just vibe-check your llm app!** Systematically evaluate and track your
+LLM experiments with TruLens. As you develop your app including prompts, models,
+retreivers, knowledge sources and more, *TruLens-Eval* is the tool you need to
+understand its performance.
+
+Fine-grained, stack-agnostic instrumentation and comprehensive evaluations help
+you to identify failure modes & systematically iterate to improve your
+application.
+
+Read more about the core concepts behind TruLens including [Feedback Functions](https://www.trulens.org/trulens_eval/getting_started/core_concepts/feedback_functions/), 
+[The RAG Triad](https://www.trulens.org/trulens_eval/getting_started/core_concepts/rag_triad/),
+and [Honest, Harmless and Helpful Evals](https://www.trulens.org/trulens_eval/getting_started/core_concepts/honest_harmless_helpful_evals/).
+
+## TruLens in the development workflow
+
+Build your first prototype then connect instrumentation and logging with
+TruLens. Decide what feedbacks you need, and specify them with TruLens to run
+alongside your app. Then iterate and compare versions of your app in an
+easy-to-use user interface 👇
+
+![Architecture
+Diagram](https://www.trulens.org/assets/images/TruLens_Architecture.png)
+
+## Installation and Setup
+
+Install the trulens-eval pip package from PyPI.
+
+```bash
+    pip install trulens-eval
+```
+
+## Quick Usage
+
+Walk through how to instrument and evaluate a RAG built from scratch with
+TruLens.
+
+[![Open In
+Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/quickstart.ipynb)
+
+### 💡 Contributing
+
+Interested in contributing? See our [contributing
+guide](https://www.trulens.org/trulens_eval/contributing/) for more details.
+<!---
+end of docs/trulens_eval/intro.md
+-->

trulens_eval-0.33.0/setup.cfg

@@ -0,0 +1,20 @@
+[metadata]
+name = trulens_eval
+version = attr: trulens_eval.__version__
+url = https://www.trulens.org
+license = MIT
+author = Truera Inc
+author_email = all@truera.com
+description = Library to systematically track and evaluate LLM based applications.
+long_description = file: README.md
+long_description_content_type = text/markdown
+classifiers = 
+	Programming Language :: Python :: 3
+	Operating System :: OS Independent
+	Development Status :: 3 - Alpha
+	License :: OSI Approved :: MIT License
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

trulens_eval-0.33.0/setup.py

@@ -0,0 +1,72 @@
+"""
+# _TruLens-Eval_ build script
+
+To build:
+
+```bash
+python setup.py bdist_wheel
+```
+
+TODO: It is more standard to configure a lot of things we configure
+here in a setup.cfg file instead. It is unclear whether we can do everything
+with a config file though so we may need to keep this script or parts of it.
+"""
+
+import os
+
+from pip._internal.req import parse_requirements
+from setuptools import find_namespace_packages
+from setuptools import setup
+from setuptools.command.build import build
+from setuptools.logging import logging
+
+required_packages = list(
+    map(
+        lambda pip_req: str(pip_req.requirement),
+        parse_requirements("trulens_eval/requirements.txt", session=None)
+    )
+)
+optional_packages = list(
+    map(
+        lambda pip_req: str(pip_req.requirement),
+        parse_requirements(
+            "trulens_eval/requirements.optional.txt", session=None
+        )
+    )
+)
+
+
+class BuildJavascript(build):
+
+    def run(self):
+        """Custom build command to run npm commands before building the package.
+    
+        This builds the record timeline component for the dashboard.
+        """
+
+        logging.info("running npm i")
+        os.system("npm i --prefix trulens_eval/react_components/record_viewer")
+        logging.info("running npm run build")
+        os.system(
+            "npm run --prefix trulens_eval/react_components/record_viewer build"
+        )
+        build.run(self)
+
+
+setup(
+    name="trulens_eval",
+    cmdclass={
+        'build': BuildJavascript,
+    },
+    include_package_data=True,  # includes things specified in MANIFEST.in
+    packages=find_namespace_packages(
+        include=["trulens_eval", "trulens_eval.*"]
+    ),
+    python_requires='>= 3.8, < 3.13',
+    entry_points={
+        'console_scripts': [
+            'trulens-eval=trulens_eval.utils.command_line:main'
+        ],
+    },
+    install_requires=required_packages
+)

trulens_eval-0.33.0/trulens_eval/Example_TruBot.py

@@ -0,0 +1,172 @@
+import os
+
+os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
+
+from langchain.chains import ConversationalRetrievalChain
+from langchain.memory import ConversationSummaryBufferMemory
+import numpy as np
+import streamlit as st
+
+from trulens_eval import feedback
+from trulens_eval import Select
+from trulens_eval import tru
+from trulens_eval import tru_chain
+from trulens_eval.feedback import Feedback
+from trulens_eval.keys import check_keys
+from trulens_eval.utils.imports import OptionalImports
+from trulens_eval.utils.imports import REQUIREMENT_PINECONE
+
+with OptionalImports(messages=REQUIREMENT_PINECONE) as opt:
+    from langchain.embeddings.openai import OpenAIEmbeddings
+    import langchain_community
+    from langchain_community.callbacks import get_openai_callback
+    from langchain_community.llms import OpenAI
+    import pinecone
+
+opt.assert_installed(mods=[pinecone, langchain_community, OpenAIEmbeddings])
+
+check_keys("OPENAI_API_KEY", "PINECONE_API_KEY", "PINECONE_ENV")
+
+# Set up GPT-3 model
+model_name = "gpt-3.5-turbo"
+
+app_id = "TruBot"
+# app_id = "TruBot_langprompt"
+# app_id = "TruBot_relevance"
+
+# Pinecone configuration.
+pinecone_client = pinecone.Pinecone(
+    api_key=os.environ.get("PINECONE_API_KEY"),  # find at app.pinecone.io
+    environment=os.environ.get("PINECONE_ENV")  # next to api key in console
+)
+
+identity = lambda h: h
+
+hugs = feedback.Huggingface()
+openai = feedback.OpenAI()
+
+# Language match between question/answer.
+f_lang_match = Feedback(hugs.language_match).on_input_output()
+# By default this will evaluate feedback on main app input and main app output.
+
+# Question/answer relevance between overall question and answer.
+f_qa_relevance = Feedback(openai.relevance).on_input_output()
+# By default this will evaluate feedback on main app input and main app output.
+
+# Question/statement relevance between question and each context chunk.
+f_qs_relevance = feedback.Feedback(openai.qs_relevance).on_input().on(
+    Select.Record.app.combine_docs_chain._call.args.inputs.input_documents[:].
+    page_content
+).aggregate(np.min)
+
+# First feedback argument is set to main app input, and the second is taken from
+# the context sources as passed to an internal `combine_docs_chain._call`.
+
+
+# @st.cache_data
+def generate_response(prompt):
+    # Embedding needed for Pinecone vector db.
+    embedding = OpenAIEmbeddings(model='text-embedding-ada-002')  # 1536 dims
+
+    # TODO: Check updated usage here.
+    docsearch = pinecone_client.from_existing_index(
+        index_name="llmdemo", embedding=embedding
+    )
+    retriever = docsearch.as_retriever()
+
+    # LLM for completing prompts, and other tasks.
+    llm = OpenAI(temperature=0, max_tokens=128)
+
+    # Conversation memory.
+    memory = ConversationSummaryBufferMemory(
+        max_token_limit=650,
+        llm=llm,
+        memory_key="chat_history",
+        output_key='answer'
+    )
+
+    # Conversational chain puts it all together.
+    chain = ConversationalRetrievalChain.from_llm(
+        llm=llm,
+        retriever=retriever,
+        return_source_documents=True,
+        memory=memory,
+        get_chat_history=identity,
+        max_tokens_limit=4096
+    )
+
+    # Language mismatch fix:
+    if "langprompt" in app_id:
+        chain.combine_docs_chain.llm_chain.prompt.template = \
+            "Use the following pieces of CONTEXT to answer the question at the end " \
+            "in the same language as the question. If you don't know the answer, " \
+            "just say that you don't know, don't try to make up an answer.\n" \
+            "\n" \
+            "CONTEXT: {context}\n" \
+            "\n" \
+            "Question: {question}\n" \
+            "Helpful Answer: "
+
+    elif "relevance" in app_id:
+        # Contexts fix
+        chain.combine_docs_chain.llm_chain.prompt.template = \
+            "Use only the relevant contexts to answer the question at the end " \
+            ". Some pieces of context may not be relevant. If you don't know the answer, " \
+            "just say that you don't know, don't try to make up an answer.\n" \
+            "\n" \
+            "Contexts: \n" \
+            "{context}\n" \
+            "\n" \
+            "Question: {question}\n" \
+            "Helpful Answer: "
+
+        # space is important
+
+        chain.combine_docs_chain.document_prompt.template = "\tContext: {page_content}"
+
+    # Trulens instrumentation.
+    tc_recorder = tru_chain.TruChain(chain, app_id=app_id)
+    with tc_recorder as recording:
+        resp = chain(dict(question=prompt))
+    tru_record = recording.records[0]
+    return tc_recorder, (resp, tru_record)
+
+
+# Set up Streamlit app
+st.title("TruBot")
+user_input = st.text_input("Ask a question about TruEra")
+
+if user_input:
+    # Generate GPT-3 response
+    prompt_input = user_input
+    # add context manager to capture tokens and cost of the chain
+
+    with get_openai_callback() as cb:
+        chain, (response, record) = generate_response(prompt_input)
+        total_tokens = cb.total_tokens
+        total_cost = cb.total_cost
+
+    answer = response['answer']
+
+    # Display response
+    st.write(answer)
+
+    record_id = tru.add_data(
+        app_id=app_id,
+        prompt=prompt_input,
+        response=answer,
+        record=record,
+        tags='dev',
+        total_tokens=total_tokens,
+        total_cost=total_cost
+    )
+
+    # Run feedback function and get value
+    feedbacks = tru.run_feedback_functions(
+        app=app,
+        record=record,
+        feedback_functions=[f_lang_match, f_qa_relevance, f_qs_relevance]
+    )
+
+    # Add value to database
+    tru.add_feedback(record_id, feedbacks)

trulens_eval-0.33.0/trulens_eval/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Truera, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

trulens_eval-0.33.0/trulens_eval/Leaderboard.py

@@ -0,0 +1,150 @@
+import asyncio
+import json
+import math
+
+# https://github.com/jerryjliu/llama_index/issues/7244:
+asyncio.set_event_loop(asyncio.new_event_loop())
+
+from millify import millify
+import streamlit as st
+from streamlit_extras.switch_page_button import switch_page
+
+from trulens_eval import Tru
+from trulens_eval.database import base as mod_db
+from trulens_eval.database.legacy.migration import MIGRATION_UNKNOWN_STR
+from trulens_eval.utils.streamlit import init_from_args
+from trulens_eval.ux import styles
+from trulens_eval.ux.components import draw_metadata
+from trulens_eval.ux.page_config import set_page_config
+from trulens_eval.ux.styles import CATEGORY
+
+if __name__ == "__main__":
+    # If not imported, gets args from command line and creates Tru singleton
+    init_from_args()
+
+
+def leaderboard():
+    """Render the leaderboard page."""
+
+    set_page_config(page_title="Leaderboard")
+
+    tru = Tru(
+    )  # get singletone whether this file was imported or executed from command line.
+
+    lms = tru.db
+
+    # Set the title and subtitle of the app
+    st.title("App Leaderboard")
+    st.write(
+        "Average feedback values displayed in the range from 0 (worst) to 1 (best)."
+    )
+    df, feedback_col_names = lms.get_records_and_feedback([])
+    feedback_defs = lms.get_feedback_defs()
+    feedback_directions = {
+        (
+            row.feedback_json.get("supplied_name", "") or
+            row.feedback_json["implementation"]["name"]
+        ): row.feedback_json.get("higher_is_better", True)
+        for _, row in feedback_defs.iterrows()
+    }
+
+    if df.empty:
+        st.write("No records yet...")
+        return
+
+    df = df.sort_values(by="app_id")
+
+    if df.empty:
+        st.write("No records yet...")
+
+    apps = list(df.app_id.unique())
+    st.markdown("""---""")
+
+    for app in apps:
+        app_df = df.loc[df.app_id == app]
+        if app_df.empty:
+            continue
+        app_str = app_df["app_json"].iloc[0]
+        app_json = json.loads(app_str)
+        metadata = app_json.get("metadata")
+        # st.text('Metadata' + str(metadata))
+        st.header(app, help=draw_metadata(metadata))
+        app_feedback_col_names = [
+            col_name for col_name in feedback_col_names
+            if not app_df[col_name].isna().all()
+        ]
+        col1, col2, col3, col4, *feedback_cols, col99 = st.columns(
+            5 + len(app_feedback_col_names)
+        )
+        latency_mean = (
+            app_df["latency"].
+            apply(lambda td: td if td != MIGRATION_UNKNOWN_STR else None).mean()
+        )
+
+        # app_df_feedback = df.loc[df.app_id == app]
+
+        col1.metric("Records", len(app_df))
+        col2.metric(
+            "Average Latency (Seconds)",
+            (
+                f"{millify(round(latency_mean, 5), precision=2)}"
+                if not math.isnan(latency_mean) else "nan"
+            ),
+        )
+        col3.metric(
+            "Total Cost (USD)",
+            f"${millify(round(sum(cost for cost in app_df.total_cost if cost is not None), 5), precision = 2)}",
+        )
+        col4.metric(
+            "Total Tokens",
+            millify(
+                sum(
+                    tokens for tokens in app_df.total_tokens
+                    if tokens is not None
+                ),
+                precision=2
+            ),
+        )
+
+        for i, col_name in enumerate(app_feedback_col_names):
+            mean = app_df[col_name].mean()
+
+            st.write(
+                styles.stmetricdelta_hidearrow,
+                unsafe_allow_html=True,
+            )
+
+            higher_is_better = feedback_directions.get(col_name, True)
+
+            if "distance" in col_name:
+                feedback_cols[i].metric(
+                    label=col_name,
+                    value=f"{round(mean, 2)}",
+                    delta_color="normal"
+                )
+            else:
+                cat = CATEGORY.of_score(mean, higher_is_better=higher_is_better)
+                feedback_cols[i].metric(
+                    label=col_name,
+                    value=f"{round(mean, 2)}",
+                    delta=f"{cat.icon} {cat.adjective}",
+                    delta_color=(
+                        "normal" if cat.compare(
+                            mean, CATEGORY.PASS[cat.direction].threshold
+                        ) else "inverse"
+                    ),
+                )
+
+        with col99:
+            if st.button("Select App", key=f"app-selector-{app}"):
+                st.session_state.app = app
+                switch_page("Evaluations")
+
+        # with st.expander("Model metadata"):
+        #    st.markdown(draw_metadata(metadata))
+
+        st.markdown("""---""")
+
+
+if __name__ == "__main__":
+    leaderboard()