vectara-agentic 0.1.10__py3-none-any.whl → 0.1.12__py3-none-any.whl

vectara_agentic/__init__.py

@@ -3,7 +3,7 @@ vectara_agentic package.
 """
 
 # Define the package version
-__version__ = "0.1.10"
+__version__ = "0.1.12"
 
 # Import classes and functions from modules
 # from .module1 import Class1, function1

vectara_agentic/_observability.py

@@ -0,0 +1,82 @@
+import os
+import json
+import pandas as pd
+
+from phoenix.otel import register
+import phoenix as px
+from openinference.instrumentation.llama_index import LlamaIndexInstrumentor
+from phoenix.trace.dsl import SpanQuery
+from phoenix.trace import SpanEvaluations
+
+from .types import ObserverType
+
+def setup_observer():
+    observer = ObserverType(os.getenv("VECTARA_AGENTIC_OBSERVER_TYPE", "NO_OBSERVER"))
+    if observer == ObserverType.ARIZE_PHOENIX:
+        phoenix_endpoint = os.getenv("PHOENIX_ENDPOINT", None)
+        if not phoenix_endpoint:
+            px.launch_app()
+            tracer_provider = register(endpoint='http://localhost:6006/v1/traces', project_name="vectara-agentic")
+        elif 'app.phoenix.arize.com' in phoenix_endpoint:   # hosted on Arizze
+            phoenix_api_key = os.getenv("PHOENIX_API_KEY", None)
+            if not phoenix_api_key:
+                raise Exception("Arize Phoenix API key not set. Please set PHOENIX_API_KEY environment variable.")
+            os.environ["PHOENIX_CLIENT_HEADERS"] = f"api_key={phoenix_api_key}"
+            os.environ["PHOENIX_COLLECTOR_ENDPOINT"] = "https://app.phoenix.arize.com"
+            tracer_provider = register(endpoint=phoenix_endpoint, project_name="vectara-agentic")
+        else:       # Self hosted Phoenix
+            tracer_provider = register(endpoint=phoenix_endpoint, project_name="vectara-agentic")
+        LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider)
+    else:
+        print("No observer set.")
+
+
+def _extract_fcs_value(output):
+    try:
+        output_json = json.loads(output)
+        if 'metadata' in output_json and 'fcs' in output_json['metadata']:
+            return output_json['metadata']['fcs']
+    except json.JSONDecodeError:
+        print(f"Failed to parse JSON: {output}")
+    except KeyError:
+        print(f"'fcs' not found in: {output_json}")
+    return None
+
+def _find_top_level_parent_id(row, all_spans):
+    current_id = row['parent_id']
+    while current_id is not None:
+        parent_row = all_spans[all_spans.index == current_id]
+        if parent_row.empty:
+            break
+        new_parent_id = parent_row['parent_id'].iloc[0]
+        if new_parent_id == current_id:
+            break
+        if new_parent_id is None:
+            return current_id
+        current_id = new_parent_id
+    return current_id
+
+def eval_fcs():
+    query = SpanQuery().select(
+        "output.value",
+        "parent_id",
+        "name"
+    )
+    client = px.Client()
+    all_spans = client.query_spans(query, project_name="vectara-agentic")
+    vectara_spans = all_spans[all_spans['name'] == 'VectaraQueryEngine._query'].copy()
+    vectara_spans['top_level_parent_id'] = vectara_spans.apply(lambda row: _find_top_level_parent_id(row, all_spans), axis=1)
+    vectara_spans['score'] = vectara_spans['output.value'].apply(lambda x: _extract_fcs_value(x))
+    
+    vectara_spans.reset_index(inplace=True)
+    top_level_spans = vectara_spans.copy()
+    top_level_spans['context.span_id'] = top_level_spans['top_level_parent_id']
+    vectara_spans = pd.concat([vectara_spans, top_level_spans], ignore_index=True)
+    vectara_spans.set_index('context.span_id', inplace=True)
+
+    px.Client().log_evaluations(
+        SpanEvaluations(
+            dataframe=vectara_spans,
+            eval_name="Vectara FCS",
+        ),
+    )

vectara_agentic/agent.py

@@ -23,17 +23,14 @@ from llama_index.core.callbacks import CallbackManager, TokenCountingHandler
 from llama_index.core.callbacks.base_handler import BaseCallbackHandler
 from llama_index.agent.openai import OpenAIAgent
 from llama_index.core.memory import ChatMemoryBuffer
-from llama_index.core import set_global_handler
-from llama_index.core.tools.types import ToolMetadata
-
-import phoenix as px
 
 from dotenv import load_dotenv
 
-from .types import AgentType, AgentStatusType, LLMRole, ObserverType, ToolType
+from .types import AgentType, AgentStatusType, LLMRole, ToolType
 from .utils import get_llm, get_tokenizer_for_model
 from ._prompts import REACT_PROMPT_TEMPLATE, GENERAL_PROMPT_TEMPLATE
 from ._callback import AgentCallbackHandler
+from ._observability import setup_observer, eval_fcs
 from .tools import VectaraToolFactory, VectaraTool
 
 
@@ -78,7 +75,7 @@ class Agent:
         custom_instructions: str = "",
         verbose: bool = True,
         update_func: Optional[Callable[[AgentStatusType, str], None]] = None,
-        agent_type: AgentType = AgentType(os.getenv("VECTARA_AGENTIC_AGENT_TYPE", "OPENAI")),
+        agent_type: AgentType = None,
     ) -> None:
         """
         Initialize the agent with the specified type, tools, topic, and system message.
@@ -91,7 +88,7 @@ class Agent:
             verbose (bool, optional): Whether the agent should print its steps. Defaults to True.
             update_func (Callable): A callback function the code calls on any agent updates.
         """
-        self.agent_type = agent_type
+        self.agent_type = agent_type or AgentType(os.getenv("VECTARA_AGENTIC_AGENT_TYPE", "OPENAI"))
         self.tools = tools
         self.llm = get_llm(LLMRole.MAIN)
         self._custom_instructions = custom_instructions
@@ -144,17 +141,10 @@ class Agent:
         else:
             raise ValueError(f"Unknown agent type: {self.agent_type}")
 
-        observer = ObserverType(os.getenv("VECTARA_AGENTIC_OBSERVER_TYPE", "NO_OBSERVER"))
-        if observer == ObserverType.ARIZE_PHOENIX:
-            if os.environ.get("OTEL_EXPORTER_OTLP_HEADERS", None):
-                set_global_handler("arize_phoenix", endpoint="https://llamatrace.com/v1/traces")
-                print("Arize Phoenix observer set. https://llamatrace.com")
-            else:
-                px.launch_app()
-                set_global_handler("arize_phoenix", endpoint="http://localhost:6006/v1/traces")
-                print("Arize Phoenix observer set. http://localhost:6006/.")
-        else:
-            print("No observer set.")
+        try:
+            setup_observer()
+        except Exception as e:
+            print(f"Failed to set up observer ({e}), ignoring")
 
     def __eq__(self, other):
         if not isinstance(other, Agent):
@@ -203,6 +193,7 @@ class Agent:
         custom_instructions: str = "",
         verbose: bool = True,
         update_func: Optional[Callable[[AgentStatusType, str], None]] = None,
+        agent_type: AgentType = None,
     ) -> "Agent":
         """
         Create an agent from tools, agent type, and language model.
@@ -219,7 +210,7 @@ class Agent:
         Returns:
             Agent: An instance of the Agent class.
         """
-        return cls(tools, topic, custom_instructions, verbose, update_func)
+        return cls(tools, topic, custom_instructions, verbose, update_func, agent_type)
 
     @classmethod
     def from_corpus(
@@ -355,6 +346,7 @@ class Agent:
             agent_response = self.agent.chat(prompt)
             if self.verbose:
                 print(f"Time taken: {time.time() - st}")
+            eval_fcs()
             return agent_response.response
         except Exception as e:
             import traceback

vectara_agentic/types.py

@@ -28,6 +28,7 @@ class ModelProvider(Enum):
     GROQ = "GROQ"
     FIREWORKS = "FIREWORKS"
     COHERE = "COHERE"
+    GEMINI = "GEMINI"
 
 
 class AgentStatusType(Enum):

vectara_agentic/utils.py

@@ -11,6 +11,7 @@ from llama_index.llms.together import TogetherLLM
 from llama_index.llms.groq import Groq
 from llama_index.llms.fireworks import Fireworks
 from llama_index.llms.cohere import Cohere
+from llama_index.llms.gemini import Gemini
 
 import tiktoken
 from typing import Tuple, Callable, Optional
@@ -24,6 +25,7 @@ provider_to_default_model_name = {
     ModelProvider.GROQ: "llama-3.1-70b-versatile",
     ModelProvider.FIREWORKS: "accounts/fireworks/models/firefunction-v2",
     ModelProvider.COHERE: "command-r-plus",
+    ModelProvider.GEMINI: "models/gemini-pro",
 }
 
 DEFAULT_MODEL_PROVIDER = ModelProvider.OPENAI
@@ -77,17 +79,19 @@ def get_llm(role: LLMRole) -> LLM:
     model_provider, model_name = _get_llm_params_for_role(role)
 
     if model_provider == ModelProvider.OPENAI:
-        llm = OpenAI(model=model_name, temperature=0)
+        llm = OpenAI(model=model_name, temperature=0, is_function_calling_model=True)
     elif model_provider == ModelProvider.ANTHROPIC:
-        llm = Anthropic(model=model_name, temperature=0)
+        llm = Anthropic(model=model_name, temperature=0, is_function_calling_model=True)
+    elif model_provider == ModelProvider.GEMINI:
+        llm = Gemini(model=model_name, temperature=0, is_function_calling_model=True)
     elif model_provider == ModelProvider.TOGETHER:
-        llm = TogetherLLM(model=model_name, temperature=0)
+        llm = TogetherLLM(model=model_name, temperature=0, is_function_calling_model=True)
     elif model_provider == ModelProvider.GROQ:
-        llm = Groq(model=model_name, temperature=0)
+        llm = Groq(model=model_name, temperature=0, is_function_calling_model=True)
     elif model_provider == ModelProvider.FIREWORKS:
-        llm = Fireworks(model=model_name, temperature=0)
+        llm = Fireworks(model=model_name, temperature=0, is_function_calling_model=True)
     elif model_provider == ModelProvider.COHERE:
-        llm = Cohere(model=model_name, temperature=0)
+        llm = Cohere(model=model_name, temperature=0, is_function_calling_model=True)
     else:
         raise ValueError(f"Unknown LLM provider: {model_provider}")
 

{vectara_agentic-0.1.10.dist-info → vectara_agentic-0.1.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vectara_agentic
-Version: 0.1.10
+Version: 0.1.12
 Summary: A Python package for creating AI Assistants and AI Agents with Vectara
 Home-page: https://github.com/vectara/py-vectara-agentic
 Author: Ofer Mendelevitch
@@ -16,23 +16,27 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: llama-index ==0.11.10
+Requires-Dist: llama-index ==0.11.13
 Requires-Dist: llama-index-indices-managed-vectara ==0.2.2
 Requires-Dist: llama-index-agent-llm-compiler ==0.2.0
-Requires-Dist: llama-index-agent-openai ==0.3.1
-Requires-Dist: llama-index-llms-openai ==0.2.7
+Requires-Dist: llama-index-agent-openai ==0.3.4
+Requires-Dist: llama-index-llms-openai ==0.2.9
 Requires-Dist: llama-index-llms-anthropic ==0.3.1
 Requires-Dist: llama-index-llms-together ==0.2.0
 Requires-Dist: llama-index-llms-groq ==0.2.0
 Requires-Dist: llama-index-llms-fireworks ==0.2.0
 Requires-Dist: llama-index-llms-cohere ==0.3.0
+Requires-Dist: llama-index-llms-gemini ==0.3.5
 Requires-Dist: llama-index-tools-yahoo-finance ==0.2.0
 Requires-Dist: llama-index-tools-arxiv ==0.2.0
 Requires-Dist: llama-index-tools-database ==0.2.0
 Requires-Dist: llama-index-tools-google ==0.2.0
 Requires-Dist: llama-index-tools-tavily-research ==0.2.0
-Requires-Dist: llama-index-callbacks-arize-phoenix ==0.2.1
-Requires-Dist: pydantic ==2.8.2
+Requires-Dist: openinference-instrumentation-llama-index ==3.0.2
+Requires-Dist: opentelemetry-proto ==1.16.0
+Requires-Dist: arize-phoenix-otel ==0.5.1
+Requires-Dist: tokenizers >=0.20
+Requires-Dist: pydantic ==2.9.2
 Requires-Dist: retrying ==1.3.4
 Requires-Dist: pymongo ==4.6.3
 Requires-Dist: python-dotenv ==1.0.1
@@ -65,7 +69,7 @@ Requires-Dist: dill ==0.3.8
 - [Vectara account](https://console.vectara.com/signup/?utm_source=github&utm_medium=code&utm_term=DevRel&utm_content=vectara-agentic&utm_campaign=github-code-DevRel-vectara-agentic)
 - A Vectara corpus with an [API key](https://docs.vectara.com/docs/api-keys)
 - [Python 3.10 or higher](https://www.python.org/downloads/)
-- OpenAI API key (or API keys for Anthropic, TOGETHER.AI, Fireworks AI, Cohere or GROQ)
+- OpenAI API key (or API keys for Anthropic, TOGETHER.AI, Fireworks AI, Cohere, GEMINI or GROQ)
 
 ## Installation
 
@@ -142,7 +146,7 @@ financial_bot_instructions = """
 Configure `vectara-agentic` using environment variables:
 
 - `VECTARA_AGENTIC_AGENT_TYPE`: valid values are `REACT`, `LLMCOMPILER` or `OPENAI` (default: `OPENAI`)
-- `VECTARA_AGENTIC_MAIN_LLM_PROVIDER`: valid values are `OPENAI`, `ANTHROPIC`, `TOGETHER`, `GROQ`, `COHERE` or `FIREWORKS` (default: `OPENAI`)
+- `VECTARA_AGENTIC_MAIN_LLM_PROVIDER`: valid values are `OPENAI`, `ANTHROPIC`, `TOGETHER`, `GROQ`, `COHERE`, `GEMINI` or `FIREWORKS` (default: `OPENAI`)
 - `VECTARA_AGENTIC_MAIN_MODEL_NAME`: agent model name (default depends on provider)
 - `VECTARA_AGENTIC_TOOL_LLM_PROVIDER`: tool LLM provider (default: `OPENAI`)
 - `VECTARA_AGENTIC_TOOL_MODEL_NAME`: tool model name (default depends on provider)
@@ -192,13 +196,20 @@ The `Agent` class supports serialization. Use the `dumps()` to serialize and `lo
 ## Observability
 
 vectara-agentic supports observability via the existing integration of LlamaIndex and Arize Phoenix.
-To enable tracing of your vectara-agentic assistant, follow these steps (adapted from [here](https://docs.llamaindex.ai/en/stable/module_guides/observability/)):
-1. Go to `https://llamatrace.com/login` an create an account, then create an API key and put it in the `PHOENIX_API_KEY` variable
-2. `os["VECTARA_AGENTIC_OBSERVER_TYPE"] = "ARIZE_PHOENIX"`: to enable Arize Phoenix observability
-3. `os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"api_key={PHOENIX_API_KEY}"`
+First, set `os["VECTARA_AGENTIC_OBSERVER_TYPE"] = "ARIZE_PHOENIX"`.
+Then you can use Arize Phoenix in three ways: 
+1. **Locally**. 
+   1. If you have a local phoenix server that you've run using e.g. `python -m phoenix.server.main serve`, vectara-agentic will send all traces to it.
+   2. If not, vectara-agentic will run a local instance during the agent's lifecycle, and will close it when finished.
+   3. In both cases, traces will be sent to the local instance, and you can see the dashboard at `http://localhost:6006`
+2. **Hosted Instance**. In this case the traces are sent to the Phoenix instances hosted on Arize.
+   1. Go to `https://app.phoenix.arize.com`, setup an account if you don't have one.
+   2. create an API key and put it in the `PHOENIX_API_KEY` variable. This variable indicates you want to use the hosted version.
+   3. To view the traces go to `https://app.phoenix.arize.com`.
+
+Now when you run your agent, all call traces are sent to Phoenix and recorded. 
+In addition, vectara-agentic also records `FCS` values into Arize for every Vectara RAG call. You can see those results in the `Feedback` column of the arize UI.
 
-Now when you run your agent, all metrics are sent to LlamaTrace and recorded. You can view them at `https://llamatrace.com`.
-If you do not include the `OTEL_EXPORTER_OTLP_HEADERS` a local instance of Arize Phoenix will be setup instead and you can view it at `http://localhost:6006`
 
 ## About Custom Instructions
 

vectara_agentic-0.1.12.dist-info/RECORD

@@ -0,0 +1,14 @@
+vectara_agentic/__init__.py,sha256=jCwV8s1vQMRav_ZDkG32Va_hjXaS52ejqq05IR0T5YY,449
+vectara_agentic/_callback.py,sha256=_o8XK1gBmsqpsJACAdJtbtnOnhLe6ZbGahCgb3WMuJQ,3674
+vectara_agentic/_observability.py,sha256=ABn3gY18QrVZZRap23Qw6xeMDBn8U7tQycja6ZNFX-Y,3338
+vectara_agentic/_prompts.py,sha256=UV03GBdz0LplkyOacJyBLbrBpWSqUS7iRtM5xmJ0BVU,4572
+vectara_agentic/agent.py,sha256=8--LXTqBZQbwQx8d6LMxcgJDPJnXev6RfeSU4FPJqQM,19172
+vectara_agentic/tools.py,sha256=1-1GmOQPrxbS6kumFrco2kGheIz0f19sgzPtb_-GjIU,19071
+vectara_agentic/tools_catalog.py,sha256=RByoXkF1GhY0rPQGLIeiqQo-j7o1h3lA6KY55ZM9mGg,4448
+vectara_agentic/types.py,sha256=SSo4Ph6aaEELF0OTKetyghq5GNgMnVCj49i8yq5TYsU,1102
+vectara_agentic/utils.py,sha256=lWfMqUhcT80jjDQOcx41I3L88z5yEAAyBQrtImnF5o0,3757
+vectara_agentic-0.1.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vectara_agentic-0.1.12.dist-info/METADATA,sha256=6iA3xwkxP_I-OpMRlN8DixSdfO54g3ZJj5TnhrbCfMI,11466
+vectara_agentic-0.1.12.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+vectara_agentic-0.1.12.dist-info/top_level.txt,sha256=qT7JB9Xz7byehzlPd_rY4WWEAvPMhs63WMWgPsFthxU,16
+vectara_agentic-0.1.12.dist-info/RECORD,,

vectara_agentic-0.1.10.dist-info/RECORD

@@ -1,13 +0,0 @@
-vectara_agentic/__init__.py,sha256=K7BcM2qRPyrdggQvXrdcPkharv0_8k763NZI3icrGxw,449
-vectara_agentic/_callback.py,sha256=_o8XK1gBmsqpsJACAdJtbtnOnhLe6ZbGahCgb3WMuJQ,3674
-vectara_agentic/_prompts.py,sha256=UV03GBdz0LplkyOacJyBLbrBpWSqUS7iRtM5xmJ0BVU,4572
-vectara_agentic/agent.py,sha256=N3FUcpvrCcnORNw57HVDAtPi_cMqqxOr7po37ybSONA,19696
-vectara_agentic/tools.py,sha256=1-1GmOQPrxbS6kumFrco2kGheIz0f19sgzPtb_-GjIU,19071
-vectara_agentic/tools_catalog.py,sha256=RByoXkF1GhY0rPQGLIeiqQo-j7o1h3lA6KY55ZM9mGg,4448
-vectara_agentic/types.py,sha256=lTL3Is5W7IFyTKuEKu_VKaAsmVFVzKss_y184ayLti8,1080
-vectara_agentic/utils.py,sha256=x8nBncooXHm6gXH-A77TRVzoPGoGleO5VeYi2fVRAA4,3340
-vectara_agentic-0.1.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vectara_agentic-0.1.10.dist-info/METADATA,sha256=vopWHXs1wXL6_-6d2-RfeKq4ilGRr5uZfouy8MzTeXE,10882
-vectara_agentic-0.1.10.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-vectara_agentic-0.1.10.dist-info/top_level.txt,sha256=qT7JB9Xz7byehzlPd_rY4WWEAvPMhs63WMWgPsFthxU,16
-vectara_agentic-0.1.10.dist-info/RECORD,,