agent-spy-monitor 0.1.0__py3-none-any.whl

agent_spy/__init__.py

@@ -0,0 +1,6 @@
+# agent_spy/__init__.py
+
+from .monitor import AgentSpy, AgentSpyExtended
+from .visualizer import Visualizer
+
+__all__ = ['AgentSpy', 'AgentSpyExtended', 'Visualizer']

agent_spy/cost_calculator.py

@@ -0,0 +1,51 @@
+# agent_spy/cost_calculator.py
+
+class CostCalculator:
+    # Define the cost per token in a constant
+    COST_PER_TOKEN = 1_000_000
+
+    # Base pricing template to avoid repetition
+    BASE_PRICING = {
+        "gpt-4o": (2.50, 10.00),
+        "gpt-4o-mini": (0.150, 0.600),
+        "gpt-4": (3.00, 6.00),
+        "gpt-4o-audio-preview": (2.50, 10.00),
+        "o1-mini": (3.00, 12.00),
+        "o1-preview": (15.00, 60.00),
+        "chatgpt-4o-latest": (5.00, 15.00),
+        "gpt-4-turbo-preview": (10.00, 30.00),
+        "gpt-4-32k": (60.00, 120.00),
+        "gpt-4-turbo": (10.00, 30.00),
+        "gpt-4-vision-preview": (10.00, 30.00),
+        "gpt-3.5-turbo": (1.50, 2.00),
+        "gpt-3.5-turbo-16k": (3.00, 4.00),
+        "claude-instant-1.2": (0.163, 0.551),
+        "claude-2": (8.00, 24.00),
+        "claude-2.1": (8.00, 24.00),
+        "claude-3-haiku-20240307": (0.25, 1.25),
+        "claude-3-5-haiku-20241022": (1.00, 5.00),
+        "claude-3-opus-20240229": (15.00, 75.00),
+        "claude-3-sonnet-20240229": (3.00, 15.00),
+        "claude-3-5-sonnet-20240620": (3.00, 15.00),
+        "claude-3-5-sonnet-20241022": (3.00, 15.00),
+        "gemini-pro": (0.5, 0.5),
+        "gemini-1.0-pro": (0.5, 0.5),
+        "gemini-1.5-pro": (1.25, 5.00),
+        "gemini-1.5-flash": (0.075, 0.30),
+    }
+
+    def __init__(self, model: str):
+        if model not in self.BASE_PRICING:
+            raise ValueError(f"Model {model} not supported.")
+        self.model = model
+
+    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        # Retrieve the base pricing for the selected model
+        input_rate, output_rate = self.BASE_PRICING[self.model]
+
+        # Calculate the costs using the cost per token
+        input_cost = input_tokens * (input_rate / self.COST_PER_TOKEN)
+        output_cost = output_tokens * (output_rate / self.COST_PER_TOKEN)
+
+        # Return the total cost
+        return input_cost + output_cost

agent_spy/logger.py

@@ -0,0 +1,16 @@
+# agent_spy/logger.py
+
+import logging
+
+class Logger:
+    def __init__(self):
+        logging.basicConfig(
+            filename='agent_spy.log',
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s'
+        )
+        self.logger = logging.getLogger()
+
+    def log(self, message: str):
+        print(message)
+        self.logger.info(message)

agent_spy/monitor.py

@@ -0,0 +1,111 @@
+# agent_spy/monitor.py
+
+import time
+import os
+import psutil
+import threading
+from .cost_calculator import CostCalculator
+from .logger import Logger
+from .utils import calculate_carbon_emissions
+import tiktoken
+from .visualizer import Visualizer 
+
+class AgentSpy:
+    def __init__(self, model: str, enable_monitoring: bool = True, encoding_name: str = "cl100k_base"):
+        self.model = model
+        self.start_time = None
+        self.end_time = None
+        self.input_tokens = 0
+        self.output_tokens = 0
+        self.total_tokens = 0
+        self.cost = 0.0
+        self.cpu_usage = []
+        self.memory_usage = []
+        self.logger = Logger()
+        self.cost_calculator = CostCalculator(model)
+        self.monitoring = False
+        self.monitor_thread = None
+        self.carbon_emissions = 0.0
+        try:
+            self.tokenizer = tiktoken.encoding_for_model(model)
+        except KeyError:
+            self.tokenizer = tiktoken.get_encoding(encoding_name)  # Default to "cl100k_base" if model is not recognized
+        self.enable_monitoring = enable_monitoring
+
+    def start(self):
+        self.logger.log("Monitoring started.")
+        self.start_time = time.time()
+        if self.enable_monitoring:
+            self.monitoring = True
+            self.monitor_thread = threading.Thread(target=self._monitor_resources)
+            self.monitor_thread.start()
+
+    def end(self):
+        self.end_time = time.time()
+        if self.enable_monitoring:
+            self.monitoring = False
+            if self.monitor_thread:
+                self.monitor_thread.join()
+        self.total_time = self.end_time - self.start_time
+        self.cost = self.cost_calculator.calculate_cost(self.input_tokens, self.output_tokens)
+        self.carbon_emissions = calculate_carbon_emissions(self.total_tokens, self.model)
+        self.logger.log("Monitoring ended.")
+        self.logger.log(f"Total Time: {self.total_time:.2f} seconds")
+        self.logger.log(f"Input Tokens: {self.input_tokens}")
+        self.logger.log(f"Output Tokens: {self.output_tokens}")
+        self.logger.log(f"Total Tokens: {self.total_tokens}")
+        self.logger.log(f"Cost: ${self.cost:.6f}")
+        self.logger.log(f"CPU Usage: {self.cpu_usage}")
+        self.logger.log(f"Memory Usage: {self.memory_usage} MB")
+        self.logger.log(f"Carbon Emissions: {self.carbon_emissions:.6f} kg CO2")
+
+    def _monitor_resources(self):
+        process = psutil.Process(os.getpid())
+        while self.monitoring:
+            cpu = process.cpu_percent(interval=1)
+            mem = process.memory_info().rss / (1024 * 1024)  # in MB
+            self.cpu_usage.append(cpu)
+            self.memory_usage.append(mem)
+            time.sleep(1)
+
+    def set_token_counts(self, input_text: str, output_text: str):
+        self.input_tokens = self.count_tokens(input_text)
+        self.output_tokens = self.count_tokens(output_text)
+        self.total_tokens = self.input_tokens + self.output_tokens
+        self.logger.log(f"Input Tokens: {self.input_tokens}")
+        self.logger.log(f"Output Tokens: {self.output_tokens}")
+        self.logger.log(f"Total Tokens: {self.total_tokens}")
+
+    def set_token_usage_from_crew_output(self, crew_output):
+        """
+        Extracts token usage metrics from a CrewOutput object and sets the token counts accordingly.
+        """
+        if hasattr(crew_output, 'token_usage') and crew_output.token_usage:
+            self.input_tokens = crew_output.token_usage.prompt_tokens
+            self.output_tokens = crew_output.token_usage.completion_tokens
+            self.total_tokens = crew_output.token_usage.total_tokens
+            self.cost = self.cost_calculator.calculate_cost(self.input_tokens, self.output_tokens)
+            self.carbon_emissions = calculate_carbon_emissions(self.total_tokens, self.model)
+            self.logger.log(f"Input Tokens: {self.input_tokens}")
+            self.logger.log(f"Output Tokens: {self.output_tokens}")
+            self.logger.log(f"Total Tokens: {self.total_tokens}")
+            self.logger.log(f"Cost: ${self.cost:.6f}")
+        else:
+            self.logger.log("No token usage information available in CrewOutput.")
+
+    def count_tokens(self, text: str) -> int:
+        return len(self.tokenizer.encode(text))
+
+
+class AgentSpyExtended(AgentSpy):
+    def __init__(self, model: str, enable_monitoring: bool = True, encoding_name: str = "cl100k_base"):
+        super().__init__(model, enable_monitoring, encoding_name)
+        self.visualizer = Visualizer(self)
+
+    def visualize(self, method='cli'):
+        if method == 'cli':
+            self.visualizer.cli_summary()
+        elif method == 'streamlit':
+            self.visualizer.streamlit_dashboard()
+        else:
+            raise ValueError("Unsupported visualization method. Choose 'cli' or 'streamlit'.")

agent_spy/old_monitor.py

@@ -0,0 +1,91 @@
+# agent_spy/monitor.py
+
+import time
+import os
+import psutil
+import threading
+from .cost_calculator import CostCalculator
+from .logger import Logger
+from .utils import calculate_carbon_emissions
+import tiktoken
+from .visualizer import Visualizer
+
+class AgentSpy:
+    def __init__(self, model: str, enable_monitoring: bool = True):
+        self.model = model
+        self.start_time = None
+        self.end_time = None
+        self.input_tokens = 0
+        self.output_tokens = 0
+        self.total_tokens = 0
+        self.cost = 0.0
+        self.cpu_usage = []
+        self.memory_usage = []
+        self.logger = Logger()
+        self.cost_calculator = CostCalculator(model)
+        self.monitoring = False
+        self.monitor_thread = None
+        self.carbon_emissions = 0.0
+        self.tokenizer = tiktoken.get_encoding("cl100k_base")  # Adjust encoding as needed
+        self.enable_monitoring = enable_monitoring
+
+    def start(self):
+        self.logger.log("Monitoring started.")
+        self.start_time = time.time()
+        if self.enable_monitoring:
+            self.monitoring = True
+            self.monitor_thread = threading.Thread(target=self._monitor_resources)
+            self.monitor_thread.start()
+
+    def end(self):
+        self.end_time = time.time()
+        if self.enable_monitoring:
+            self.monitoring = False
+            if self.monitor_thread:
+                self.monitor_thread.join()
+        self.total_time = self.end_time - self.start_time
+        self.cost = self.cost_calculator.calculate_cost(self.input_tokens, self.output_tokens)
+        self.carbon_emissions = calculate_carbon_emissions(self.total_tokens, self.model)
+        self.logger.log("Monitoring ended.")
+        self.logger.log(f"Total Time: {self.total_time:.2f} seconds")
+        self.logger.log(f"Input Tokens: {self.input_tokens}")
+        self.logger.log(f"Output Tokens: {self.output_tokens}")
+        self.logger.log(f"Total Tokens: {self.total_tokens}")
+        self.logger.log(f"Cost: ${self.cost:.6f}")
+        self.logger.log(f"CPU Usage: {self.cpu_usage}")
+        self.logger.log(f"Memory Usage: {self.memory_usage} MB")
+        self.logger.log(f"Carbon Emissions: {self.carbon_emissions:.6f} kg CO2")
+
+    def _monitor_resources(self):
+        process = psutil.Process(os.getpid())
+        while self.monitoring:
+            cpu = process.cpu_percent(interval=1)
+            mem = process.memory_info().rss / (1024 * 1024)  # in MB
+            self.cpu_usage.append(cpu)
+            self.memory_usage.append(mem)
+            time.sleep(1)
+
+    def set_token_counts(self, input_text: str, output_text: str):
+        self.input_tokens = self.count_tokens(input_text)
+        self.output_tokens = self.count_tokens(output_text)
+        self.total_tokens = self.input_tokens + self.output_tokens
+        self.logger.log(f"Input Tokens: {self.input_tokens}")
+        self.logger.log(f"Output Tokens: {self.output_tokens}")
+        self.logger.log(f"Total Tokens: {self.total_tokens}")
+
+    def count_tokens(self, text: str) -> int:
+        return len(self.tokenizer.encode(text))
+
+
+class AgentSpyExtended(AgentSpy):
+    def __init__(self, model: str):
+        super().__init__(model)
+        self.visualizer = Visualizer(self)
+
+    def visualize(self, method='cli'):
+        if method == 'cli':
+            self.visualizer.cli_summary()
+        elif method == 'streamlit':
+            self.visualizer.streamlit_dashboard()
+        else:
+            raise ValueError("Unsupported visualization method. Choose 'cli' or 'streamlit'.")

agent_spy/utils.py

@@ -0,0 +1,10 @@
+# agent_spy/utils.py
+
+def calculate_carbon_emissions(total_tokens: int, model: str) -> float:
+    # Placeholder for actual carbon calculation
+    # This can be replaced with a real formula or API call
+    emissions_per_token = {
+        "gpt-4o": 0.0001,      # kg CO2 per token
+        "gpt-4o-mini": 0.00005
+    }
+    return total_tokens * emissions_per_token.get(model, 0)

agent_spy/visualizer.py

@@ -0,0 +1,49 @@
+# agent_spy/visualizer.py
+
+import matplotlib.pyplot as plt
+import streamlit as st
+
+class Visualizer:
+    def __init__(self, monitor):
+        self.monitor = monitor
+
+    def cli_summary(self):
+        print("\n--- Agent Spy Summary ---")
+        print(f"Total Time: {self.monitor.total_time:.2f} seconds")
+        print(f"Input Tokens: {self.monitor.input_tokens}")
+        print(f"Output Tokens: {self.monitor.output_tokens}")
+        print(f"Total Tokens: {self.monitor.total_tokens}")
+        print(f"Cost: ${self.monitor.cost:.6f}")
+        avg_cpu = sum(self.monitor.cpu_usage) / len(self.monitor.cpu_usage) if self.monitor.cpu_usage else 0
+        avg_mem = sum(self.monitor.memory_usage) / len(self.monitor.memory_usage) if self.monitor.memory_usage else 0
+        print(f"Average CPU Usage: {avg_cpu:.2f}%")
+        print(f"Average Memory Usage: {avg_mem:.2f} MB")
+        print(f"Carbon Emissions: {self.monitor.carbon_emissions:.6f} kg CO2")
+        print("----------------------------\n")
+
+    def streamlit_dashboard(self):
+        st.title("Agent Spy Dashboard")
+        st.header("Performance Metrics")
+        st.write(f"**Total Time:** {self.monitor.total_time:.2f} seconds")
+        st.write(f"**Input Tokens:** {self.monitor.input_tokens}")
+        st.write(f"**Output Tokens:** {self.monitor.output_tokens}")
+        st.write(f"**Total Tokens:** {self.monitor.total_tokens}")
+        st.write(f"**Cost:** ${self.monitor.cost:.6f}")
+        st.write(f"**Carbon Emissions:** {self.monitor.carbon_emissions:.6f} kg CO2")
+
+        st.header("Resource Utilization")
+        if self.monitor.cpu_usage and self.monitor.memory_usage:
+            fig, ax = plt.subplots(2, 1, figsize=(10, 6))
+            ax[0].plot(self.monitor.cpu_usage, label='CPU Usage (%)', color='blue')
+            ax[0].set_xlabel('Time (s)')
+            ax[0].set_ylabel('CPU Usage (%)')
+            ax[0].legend()
+
+            ax[1].plot(self.monitor.memory_usage, label='Memory Usage (MB)', color='orange')
+            ax[1].set_xlabel('Time (s)')
+            ax[1].set_ylabel('Memory Usage (MB)')
+            ax[1].legend()
+
+            st.pyplot(fig)
+        else:
+            st.write("No resource usage data available.")

agent_spy_monitor-0.1.0.dist-info/METADATA

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: agent-spy-monitor
+Version: 0.1.0
+Summary: An operational monitoring library for AI agent applications
+Author-email: Nidhish Wakodikar <nidhishwakodikar@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/nidhishwakodikar/agent-spy-monitor
+Project-URL: Repository, https://github.com/nidhishwakodikar/agent-spy-monitor
+Project-URL: Issues, https://github.com/nidhishwakodikar/agent-spy-monitor/issues
+Project-URL: Documentation, https://github.com/nidhishwakodikar/agent-spy-monitor#readme
+Keywords: ai,agent,monitoring,tokens,cost,performance
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Monitoring
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: matplotlib>=3.5.0
+Requires-Dist: psutil>=5.8.0
+Requires-Dist: streamlit>=1.20.0
+Requires-Dist: tiktoken>=0.4.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Dynamic: license-file
+
+## Features
+
+1. **Token Counting**
+   - Total tokens
+   - Input tokens
+   - Output tokens
+
+2. **Cost Calculation**
+   - Based on the token usage and model pricing
+
+3. **Performance Metrics**
+   - Time taken for each call
+   - CPU and memory consumption
+
+4. **Environmental Impact**
+   - Estimated CO₂ emissions
+
+5. **Logging**
+   - Comprehensive logs of all operations
+
+6. **Visualization**
+   - CLI summaries
+   - Streamlit dashboard for detailed insights
+
+## Installation
+
+You can install **Agent Watch** via [PyPI](https://pypi.org/project/agent-watch/) using `pip`:
+
+```bash
+pip install agent-watch
+```

agent_spy_monitor-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+agent_spy/__init__.py,sha256=VPKLiCnT4FpHuurmIEqHDMyIPmEAJ1exhJRcmnTWNTY,170
+agent_spy/cost_calculator.py,sha256=Q-xTEoJI8YWifd1kywEnCLrX_NaShGDhPSe7CfXCjY0,2001
+agent_spy/logger.py,sha256=b6zx4Enpi5zhB4KT2WgNtush217-YioarTyY9HvbA2E,396
+agent_spy/monitor.py,sha256=RP4TEdKyFikOmod8mwVa_ICbTd5qFL0CQbkRELiLA7c,4953
+agent_spy/old_monitor.py,sha256=Um_3tw9f4aajsZwGun4-YVKj7oA2EWn2swxOX1irNkg,3640
+agent_spy/utils.py,sha256=kKHKXZkWHokLza_b9FcVqrM7ZeA-ioPTEBBR4XdMvE4,384
+agent_spy/visualizer.py,sha256=n2eMIntGvjuN8s9dz4jwXYUABPV1heY4j2HeRJZefHk,2260
+agent_spy_monitor-0.1.0.dist-info/licenses/LICENSE,sha256=raBN8zay4QP5JYmgq8MjtDhNcnN6VMmHRA2rso99TTk,1082
+agent_spy_monitor-0.1.0.dist-info/METADATA,sha256=NZRl5ndhKCN_h7-cs1qA06Tqj9VO6M_wNmK5eGMyTp0,2348
+agent_spy_monitor-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+agent_spy_monitor-0.1.0.dist-info/top_level.txt,sha256=WT-r-wmnQU8bxivGmY0edi3x3-JDiL7Mcj-gpPFniiI,10
+agent_spy_monitor-0.1.0.dist-info/RECORD,,

agent_spy_monitor-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

agent_spy_monitor-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,10 @@
+MIT License
+
+Copyright (c) [2024] [AI Anytime]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

agent_spy_monitor-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+agent_spy