genai-otel-instrument 0.1.24__py3-none-any.whl

genai_otel/mcp_instrumentors/manager.py

@@ -0,0 +1,139 @@
+"""Manager for OpenTelemetry instrumentation of Model Context Protocol (MCP) tools.
+
+This module provides the `MCPInstrumentorManager` class, which orchestrates
+the automatic instrumentation of various MCP tools, including databases, caching
+layers, message queues, vector databases, and generic API calls. It ensures
+that these components are integrated into the OpenTelemetry tracing and metrics
+system.
+"""
+
+import asyncio
+import logging
+
+import httpx
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+
+from ..config import OTelConfig
+from .api_instrumentor import APIInstrumentor
+from .database_instrumentor import DatabaseInstrumentor
+from .kafka_instrumentor import KafkaInstrumentor
+from .redis_instrumentor import RedisInstrumentor
+from .vector_db_instrumentor import VectorDBInstrumentor
+
+logger = logging.getLogger(__name__)
+
+
+class MCPInstrumentorManager:  # pylint: disable=R0903
+    """Manager for MCP (Model Context Protocol) tool instrumentation"""
+
+    def __init__(self, config: OTelConfig):
+        self.config = config
+        self.instrumentors = []
+
+    def instrument_all(self, fail_on_error: bool = False):  # pylint: disable=R0912, R0915
+        """Instrument all detected MCP tools"""
+
+        success_count = 0
+        failure_count = 0
+
+        # HTTP/API instrumentation (disabled by default to avoid conflicts)
+        if self.config.enable_http_instrumentation:
+            try:
+                logger.info("Instrumenting HTTP/API calls")
+                # CRITICAL: Do NOT instrument requests library when using OTLP HTTP exporters
+                # RequestsInstrumentor patches requests.Session at class level, breaking OTLP exporters
+                # that use requests internally. The OTEL_PYTHON_REQUESTS_EXCLUDED_URLS doesn't help
+                # because it only works at request-time, not at instrumentation-time.
+                #
+                # TODO: Find a way to instrument user requests without breaking OTLP exporters
+                # RequestsInstrumentor().instrument()
+
+                logger.warning(
+                    "Requests library instrumentation is disabled to prevent conflicts with OTLP exporters"
+                )
+
+                # HTTPx is safe to instrument
+                HTTPXClientInstrumentor().instrument()
+                api_instrumentor = APIInstrumentor(self.config)
+                api_instrumentor.instrument(self.config)
+                logger.info("✓ HTTP/API instrumentation enabled (requests library excluded)")
+                success_count += 1
+            except ImportError as e:
+                failure_count += 1
+                logger.debug(f"✗ HTTP/API instrumentation skipped due to missing dependency: {e}")
+            except Exception as e:
+                failure_count += 1
+                logger.error(f"✗ Failed to instrument HTTP/API: {e}", exc_info=True)
+                if fail_on_error:
+                    raise
+        else:
+            logger.info("HTTP/API instrumentation disabled (enable_http_instrumentation=False)")
+
+        # Database instrumentation
+        try:
+            logger.info("Instrumenting databases")
+            db_instrumentor = DatabaseInstrumentor(self.config)
+            result = db_instrumentor.instrument()
+            if result > 0:
+                success_count += 1
+                logger.info(f"✓ Database instrumentation enabled ({result} databases)")
+        except ImportError as e:
+            failure_count += 1
+            logger.debug(f"✗ Database instrumentation skipped due to missing dependency: {e}")
+        except Exception as e:
+            failure_count += 1
+            logger.error(f"✗ Failed to instrument databases: {e}", exc_info=True)
+            if fail_on_error:
+                raise
+
+        # Redis instrumentation
+        try:
+            logger.info("Instrumenting Redis")
+            redis_instrumentor = RedisInstrumentor(self.config)
+            redis_instrumentor.instrument()
+            success_count += 1
+        except ImportError as e:
+            failure_count += 1
+            logger.debug(f"✗ Redis instrumentation skipped due to missing dependency: {e}")
+        except Exception as e:
+            failure_count += 1
+            logger.error(f"✗ Failed to instrument Redis: {e}", exc_info=True)
+            if fail_on_error:
+                raise
+
+        # Kafka instrumentation
+        try:
+            logger.info("Instrumenting Kafka")
+            kafka_instrumentor = KafkaInstrumentor(self.config)
+            kafka_instrumentor.instrument()
+            success_count += 1
+        except ImportError as e:
+            failure_count += 1
+            logger.debug(f"✗ Kafka instrumentation skipped due to missing dependency: {e}")
+        except Exception as e:
+            failure_count += 1
+            logger.error(f"✗ Failed to instrument Kafka: {e}", exc_info=True)
+            if fail_on_error:
+                raise
+
+        # Vector DB instrumentation
+        try:
+            logger.info("Instrumenting Vector DBs")
+            vector_db_instrumentor = VectorDBInstrumentor(self.config)
+            result = vector_db_instrumentor.instrument()
+            if result > 0:
+                success_count += 1
+                logger.info(f"✓ Vector DB instrumentation enabled ({result} databases)")
+        except ImportError as e:
+            failure_count += 1
+            logger.debug(f"✗ Vector DB instrumentation skipped due to missing dependency: {e}")
+        except Exception as e:
+            failure_count += 1
+            logger.error(f"✗ Failed to instrument Vector DBs: {e}", exc_info=True)
+            if fail_on_error:
+                raise
+
+        logger.info(
+            f"MCP instrumentation summary: {success_count} succeeded, " f"{failure_count} failed"
+        )

genai_otel/mcp_instrumentors/redis_instrumentor.py

@@ -0,0 +1,31 @@
+"""OpenTelemetry instrumentor for Redis clients.
+
+This module provides the `RedisInstrumentor` class, which automatically
+instruments Redis operations, enabling tracing of caching interactions
+within GenAI applications.
+"""
+
+import logging
+
+from opentelemetry.instrumentation.redis import RedisInstrumentor as OTelRedisInstrumentor
+
+from ..config import OTelConfig
+
+logger = logging.getLogger(__name__)
+
+
+class RedisInstrumentor:  # pylint: disable=R0903
+    """Instrument Redis clients"""
+
+    def __init__(self, config: OTelConfig):
+        self.config = config
+
+    def instrument(self):
+        """Instrument Redis"""
+        try:
+            OTelRedisInstrumentor().instrument()
+            logger.info("Redis instrumentation enabled")
+        except ImportError:
+            logger.debug("Redis-py not installed, skipping instrumentation.")
+        except Exception as e:
+            logger.warning(f"Redis instrumentation failed: {e}")

genai_otel/mcp_instrumentors/vector_db_instrumentor.py

@@ -0,0 +1,265 @@
+"""OpenTelemetry instrumentor for various vector database clients.
+
+This module provides the `VectorDBInstrumentor` class, which automatically
+instruments popular Python vector database libraries such as Pinecone, Weaviate,
+Qdrant, ChromaDB, Milvus, and FAISS, enabling tracing of vector search and
+related operations within GenAI applications.
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+import wrapt
+from opentelemetry import trace
+from opentelemetry.trace import SpanKind, Status, StatusCode
+
+from ..config import OTelConfig
+
+logger = logging.getLogger(__name__)
+
+
+class VectorDBInstrumentor:  # pylint: disable=R0903
+    """Instrument vector database clients"""
+
+    def __init__(self, config: OTelConfig):
+        self.config = config
+        self.tracer = trace.get_tracer(__name__)
+
+    def instrument(self):
+        """Instrument all detected vector DB libraries"""
+        instrumented_count = 0
+        if self._instrument_pinecone():
+            instrumented_count += 1
+        if self._instrument_weaviate():
+            instrumented_count += 1
+        if self._instrument_qdrant():
+            instrumented_count += 1
+        if self._instrument_chroma():
+            instrumented_count += 1
+        if self._instrument_milvus():
+            instrumented_count += 1
+        if self._instrument_faiss():
+            instrumented_count += 1
+        return instrumented_count
+
+    def _instrument_pinecone(self):
+        """Instrument Pinecone operations"""
+        try:
+            import pinecone
+
+            # Check Pinecone version to handle API differences
+            pinecone_version = getattr(pinecone, "__version__", "0.0.0")
+
+            # Pinecone 3.0+ uses a different API structure
+            if hasattr(pinecone, "Pinecone"):
+                # New API (3.0+)
+                logger.info("Detected Pinecone 3.0+ API")
+                wrapt.wrap_function_wrapper(
+                    "pinecone", "Pinecone.__init__", self._wrap_pinecone_init
+                )
+
+            elif hasattr(pinecone, "Index"):
+                # Old API (2.x)
+                logger.info("Detected Pinecone 2.x API")
+                original_query = pinecone.Index.query
+                original_upsert = pinecone.Index.upsert
+                original_delete = pinecone.Index.delete
+
+                pinecone.Index.query = self._wrap_pinecone_method(original_query, "pinecone.query")
+                pinecone.Index.upsert = self._wrap_pinecone_method(
+                    original_upsert, "pinecone.upsert"
+                )
+                pinecone.Index.delete = self._wrap_pinecone_method(
+                    original_delete, "pinecone.delete"
+                )
+            else:
+                logger.warning("Could not detect Pinecone API version. Skipping instrumentation.")
+                return False
+
+            logger.info("Pinecone instrumentation enabled")
+            return True
+
+        except ImportError:
+            logger.info("Pinecone not installed, skipping instrumentation")
+            return False
+        except Exception as e:
+            if "pinecone-client" in str(e) and "renamed" in str(e):
+                logger.error(
+                    "Failed to instrument Pinecone: %s. Please ensure only the `pinecone` package is installed (uninstall `pinecone-client` if present).",
+                    e,
+                )
+            else:
+                logger.error(f"Failed to instrument Pinecone: {e}", exc_info=True)
+            return False
+
+    def _wrap_pinecone_init(self, wrapped, instance, args, kwargs):
+        """Wrapper for Pinecone.__init__ to instrument index methods."""
+        result = wrapped(*args, **kwargs)
+        if hasattr(instance, "Index"):
+            original_index = instance.Index
+
+            @wrapt.decorator
+            def traced_index(wrapped_idx, idx_instance, idx_args, idx_kwargs):
+                idx = wrapped_idx(*idx_args, **idx_kwargs)
+                if hasattr(idx_instance, "query"):
+                    idx_instance.query = self._wrap_pinecone_method(
+                        idx_instance.query, "pinecone.index.query"
+                    )
+                if hasattr(idx_instance, "upsert"):
+                    idx_instance.upsert = self._wrap_pinecone_method(
+                        idx_instance.upsert, "pinecone.index.upsert"
+                    )
+                if hasattr(idx_instance, "delete"):
+                    idx_instance.delete = self._wrap_pinecone_method(
+                        idx_instance.delete, "pinecone.index.delete"
+                    )
+                return idx
+
+            instance.Index = traced_index(original_index)
+        return result
+
+    def _wrap_pinecone_method(self, original_method, operation_name):
+        """Wrap a Pinecone method with tracing"""
+
+        def wrapper(*args, **kwargs):
+            tracer = trace.get_tracer(__name__)
+            with tracer.start_as_current_span(
+                operation_name,
+                kind=SpanKind.CLIENT,
+                attributes={"db.system": "pinecone", "db.operation": operation_name.split(".")[-1]},
+            ) as span:
+                try:
+                    result = original_method(*args, **kwargs)
+                    span.set_status(Status(StatusCode.OK))
+                    return result
+                except Exception as e:
+                    span.set_status(Status(StatusCode.ERROR, str(e)))
+                    span.record_exception(e)
+                    raise
+
+        return wrapper
+
+    def _instrument_weaviate(self):
+        """Instrument Weaviate"""
+        try:
+            import weaviate
+
+            @wrapt.decorator
+            def wrapped_query(wrapped, instance, args, kwargs):  # pylint: disable=W0613
+                with self.tracer.start_as_current_span("weaviate.query") as span:
+                    span.set_attribute("db.system", "weaviate")
+                    span.set_attribute("db.operation", "query")
+                    result = wrapped(*args, **kwargs)
+                    return result
+
+            weaviate.Client.query = wrapped_query(weaviate.Client.query)  # pylint: disable=E1120
+            logger.info("Weaviate instrumentation enabled")
+            return True
+
+        except ImportError:
+            return False
+
+    def _instrument_qdrant(self):
+        """Instrument Qdrant"""
+        try:
+            from qdrant_client import QdrantClient
+
+            original_search = QdrantClient.search
+
+            def wrapped_search(instance, *args, **kwargs):
+                with self.tracer.start_as_current_span("qdrant.search") as span:
+                    span.set_attribute("db.system", "qdrant")
+                    span.set_attribute("db.operation", "search")
+
+                    collection = kwargs.get("collection_name", args[0] if args else "unknown")
+                    span.set_attribute("vector.collection", collection)
+
+                    limit = kwargs.get("limit", 10)
+                    span.set_attribute("vector.limit", limit)
+
+                    result = original_search(instance, *args, **kwargs)
+                    return result
+
+            QdrantClient.search = wrapped_search
+            logger.info("Qdrant instrumentation enabled")
+            return True
+
+        except ImportError:
+            return False
+
+    def _instrument_chroma(self):
+        """Instrument ChromaDB"""
+        try:
+            import chromadb
+
+            original_query = chromadb.Collection.query
+
+            def wrapped_query(instance, *args, **kwargs):
+                with self.tracer.start_as_current_span("chroma.query") as span:
+                    span.set_attribute("db.system", "chromadb")
+                    span.set_attribute("db.operation", "query")
+                    span.set_attribute("vector.collection", instance.name)
+
+                    n_results = kwargs.get("n_results", 10)
+                    span.set_attribute("vector.n_results", n_results)
+
+                    result = original_query(instance, *args, **kwargs)
+                    return result
+
+            chromadb.Collection.query = wrapped_query
+            logger.info("ChromaDB instrumentation enabled")
+            return True
+
+        except ImportError:
+            return False
+
+    def _instrument_milvus(self):
+        """Instrument Milvus"""
+        try:
+            from pymilvus import Collection
+
+            original_search = Collection.search
+
+            def wrapped_search(instance, *args, **kwargs):
+                with self.tracer.start_as_current_span("milvus.search") as span:
+                    span.set_attribute("db.system", "milvus")
+                    span.set_attribute("db.operation", "search")
+                    span.set_attribute("vector.collection", instance.name)
+
+                    limit = kwargs.get("limit", 10)
+                    span.set_attribute("vector.limit", limit)
+
+                    result = original_search(instance, *args, **kwargs)
+                    return result
+
+            Collection.search = wrapped_search
+            logger.info("Milvus instrumentation enabled")
+            return True
+
+        except ImportError:
+            return False
+
+    def _instrument_faiss(self):
+        """Instrument FAISS"""
+        try:
+            import faiss
+
+            original_search = faiss.Index.search
+
+            def wrapped_search(instance, *args, **kwargs):
+                with self.tracer.start_as_current_span("faiss.search") as span:
+                    span.set_attribute("db.system", "faiss")
+                    span.set_attribute("db.operation", "search")
+
+                    k = args[1] if len(args) > 1 else kwargs.get("k", 10)
+                    span.set_attribute("vector.k", k)
+
+                    result = original_search(instance, *args, **kwargs)
+                    return result
+
+            faiss.Index.search = wrapped_search
+            logger.info("FAISS instrumentation enabled")
+            return True
+
+        except ImportError:
+            return False

genai_otel/metrics.py

@@ -0,0 +1,148 @@
+# isort: skip_file
+import logging
+import os
+from typing import Any, Dict, Optional, Tuple
+
+from opentelemetry import metrics
+from opentelemetry.metrics import Meter  # Import Meter here
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics._internal.export import MetricExporter
+from opentelemetry.sdk.metrics.export import (
+    ConsoleMetricExporter,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.resources import (
+    DEPLOYMENT_ENVIRONMENT,
+    SERVICE_NAME,
+    TELEMETRY_SDK_NAME,
+    Resource,
+)
+
+logger = logging.getLogger(__name__)
+
+# Correct the import for OTLP Metric Exporter
+if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") == "grpc":
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (  # noqa: E402
+        OTLPMetricExporter,
+    )
+else:
+    from opentelemetry.exporter.otlp.proto.http.metric_exporter import (  # noqa: E402
+        OTLPMetricExporter,
+    )
+
+# Global variables to hold the MeterProvider and Meter
+_meter_provider: Optional[MeterProvider] = None
+_meter: Optional[Meter] = None
+
+
+def get_meter() -> Meter:
+    """
+    Returns the globally configured Meter.
+    """
+    return metrics.get_meter(__name__)
+
+
+def get_meter_provider() -> MeterProvider:
+    """
+    Returns the globally configured MeterProvider.
+    """
+    return metrics.get_meter_provider()
+
+
+_DB_CLIENT_OPERATION_DURATION_BUCKETS = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10]
+
+_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS = [
+    0.01,
+    0.02,
+    0.04,
+    0.08,
+    0.16,
+    0.32,
+    0.64,
+    1.28,
+    2.56,
+    5.12,
+    10.24,
+    20.48,
+    40.96,
+    81.92,
+]
+
+_GEN_AI_SERVER_TBT = [
+    0.01,
+    0.025,
+    0.05,
+    0.075,
+    0.1,
+    0.15,
+    0.2,
+    0.3,
+    0.4,
+    0.5,
+    0.75,
+    1.0,
+    2.5,
+]
+
+_GEN_AI_SERVER_TFTT = [
+    0.001,
+    0.005,
+    0.01,
+    0.02,
+    0.04,
+    0.06,
+    0.08,
+    0.1,
+    0.25,
+    0.5,
+    0.75,
+    1.0,
+    2.5,
+    5.0,
+    7.5,
+    10.0,
+]
+
+_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS = [
+    1,
+    4,
+    16,
+    64,
+    256,
+    1024,
+    4096,
+    16384,
+    65536,
+    262144,
+    1048576,
+    4194304,
+    16777216,
+    67108864,
+]
+
+# MCP-specific bucket boundaries for performance and size metrics
+_MCP_CLIENT_OPERATION_DURATION_BUCKETS = [
+    0.001,  # 1ms
+    0.005,  # 5ms
+    0.01,  # 10ms
+    0.05,  # 50ms
+    0.1,  # 100ms
+    0.5,  # 500ms
+    1.0,  # 1s
+    2.0,  # 2s
+    5.0,  # 5s
+    10.0,  # 10s
+]
+
+_MCP_PAYLOAD_SIZE_BUCKETS = [
+    100,  # 100 bytes
+    500,  # 500 bytes
+    1024,  # 1KB
+    5120,  # 5KB
+    10240,  # 10KB
+    51200,  # 50KB
+    102400,  # 100KB
+    512000,  # 500KB
+    1048576,  # 1MB
+    5242880,  # 5MB
+]

genai_otel/py.typed

@@ -0,0 +1,2 @@
+# Marker file for PEP 561
+# This file indicates that the package supports type checking