MindsDB 25.1.2.1__py3-none-any.whl → 25.1.3.0__py3-none-any.whl

mindsdb/utilities/otel/logger.py

@@ -0,0 +1,25 @@
+import logging
+
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.sdk._logs._internal.export import LogExporter
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+
+from mindsdb.utilities.log import get_mindsdb_log_level
+
+
+def setup_logger(resource: Resource, exporter: LogExporter) -> None:
+    """
+    Setup OpenTelemetry logging
+    """
+    mindsdb_log_level = get_mindsdb_log_level()
+
+    logger_provider = LoggerProvider(resource=resource)
+    set_logger_provider(logger_provider)
+
+    logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+    handler = LoggingHandler(level=mindsdb_log_level, logger_provider=logger_provider)
+
+    # Attach OTLP handler to root logger
+    logging.getLogger().addHandler(handler)

mindsdb/utilities/otel/meter.py

@@ -0,0 +1,19 @@
+from opentelemetry import metrics
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.metrics.export import (
+    MetricExporter,
+    PeriodicExportingMetricReader,
+)
+
+
+def setup_meter(resource: Resource, exporter: MetricExporter) -> None:
+    """
+    Setup OpenTelemetry metrics
+    """
+
+    metric_reader = PeriodicExportingMetricReader(exporter=exporter)
+    provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
+
+    # Sets the global default meter provider
+    metrics.set_meter_provider(provider)

mindsdb/utilities/otel/metric_handlers/__init__.py

@@ -0,0 +1,25 @@
+from mindsdb.utilities.otel import metrics, OTEL_SERVICE_NAME
+
+_query_request_counter = None
+
+
+def get_query_request_counter():
+    """
+    Get the query request counter
+    """
+    global _query_request_counter
+
+    # Create the counter if it does not exist
+    if _query_request_counter is None:
+        meter_name = f"{OTEL_SERVICE_NAME}.query_service_meter"
+        
+        # Get the meter from the main metrics object
+        meter = metrics.get_meter(meter_name)
+
+        _query_request_counter = meter.create_counter(
+            name="query_request_count",
+            description="Counts the number of times the SQL query is called",
+            unit="1",
+        )
+
+    return _query_request_counter

mindsdb/utilities/otel/tracer.py

@@ -0,0 +1,16 @@
+from opentelemetry import trace
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter
+from opentelemetry.sdk.trace.sampling import Sampler
+
+
+def setup_tracer(resource: Resource, sampler: Sampler, exporter: SpanExporter) -> None:
+    """
+    Setup OpenTelemetry tracing
+    """
+    # Set the tracer provider with the custom resource
+    trace.set_tracer_provider(TracerProvider(resource=resource, sampler=sampler))
+
+    # Replace the default span processor with the custom one
+    trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(exporter))

mindsdb/utilities/utils.py

@@ -0,0 +1,34 @@
+import csv
+import re
+import typing
+
+
+def parse_csv_attributes(csv_attributes: typing.Optional[str] = "") -> typing.Dict[str, str]:
+    """
+    Parse the raw_attributes variable, which uses the CSV format:
+    key=value,another=something_else
+
+    Returns:
+        dict: Parsed key-value pairs as a dictionary.
+    """
+    attributes = {}
+
+    if not csv_attributes:
+        return attributes  # Return empty dictionary if the variable is not set
+
+    try:
+        # Use CSV reader to handle parsing the input
+        reader = csv.reader([csv_attributes])
+        for row in reader:
+            for pair in row:
+                # Match key=value pattern
+                match = re.match(r'^\s*([^=]+?)\s*=\s*(.+?)\s*$', pair)
+                if match:
+                    key, value = match.groups()
+                    attributes[key.strip()] = value.strip()
+                else:
+                    raise ValueError(f"Invalid attribute format: {pair}")
+    except Exception as e:
+        raise ValueError(f"Failed to parse csv_attributes='{csv_attributes}': {e}")
+
+    return attributes

mindsdb/utilities/otel.py

@@ -1,72 +0,0 @@
-import os
-
-from opentelemetry import trace
-from opentelemetry.sdk.resources import Resource
-from opentelemetry.sdk.trace import TracerProvider, Span
-from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
-from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
-
-from mindsdb.utilities import log
-logger = log.getLogger(__name__)
-
-
-# Check OpenTelemetry exporter type
-OTEL_EXPORTER_TYPE = os.getenv("OTEL_EXPORTER_TYPE", "console")  # console or otlp
-
-# Define OTLP endpoint. If not set, the default OTLP endpoint will be used
-OTEL_OTLP_ENDPOINT = os.getenv("OTEL_OTLP_ENDPOINT", "http://localhost:4317")
-
-# Define service name
-OTEL_SERVICE_NAME = os.getenv("OTEL_SERVICE_NAME", "mindsdb")
-
-# The name of the environment we"re on, by default local for development, this is set differently per-env in our Helm chart values files
-OTEL_SERVICE_ENVIRONMENT = os.getenv("OTEL_SERVICE_ENVIRONMENT", "local").lower()
-
-# Define service release
-OTEL_SERVICE_RELEASE = os.getenv("OTEL_SERVICE_RELEASE", "local").lower()
-
-# By default we have Open Telemetry SDK enabled on all envs, except for local which is disabled by default
-#   If you want to enable Open Telemetry on local for some reason please set OTEL_SDK_FORCE_RUN to true
-OTEL_SDK_DISABLED = os.getenv("OTEL_SDK_DISABLED", "false").lower() == "true" or os.getenv("OTEL_SERVICE_ENVIRONMENT", "local").lower() == "local"
-OTEL_SDK_FORCE_RUN = os.getenv("OTEL_SDK_FORCE_RUN", "false").lower() == "true"
-
-# Custom span processor to add global tags to spans
-
-
-class GlobalTaggingSpanProcessor(BatchSpanProcessor):
-    def on_start(self, span: Span, parent_context):
-        # Add environment and release to every span
-        span.set_attribute("environment", OTEL_SERVICE_ENVIRONMENT)
-        span.set_attribute("release", OTEL_SERVICE_RELEASE)
-        super().on_start(span, parent_context)
-
-
-if not OTEL_SDK_DISABLED or OTEL_SDK_FORCE_RUN:
-    logger.info("OpenTelemetry enabled")
-    logger.info(f"OpenTelemetry exporter type: {OTEL_EXPORTER_TYPE}")
-    logger.info(f"OpenTelemetry service name: {OTEL_SERVICE_NAME}")
-    logger.info(f"OpenTelemetry service environment: {OTEL_SERVICE_ENVIRONMENT}")
-    logger.info(f"OpenTelemetry service release: {OTEL_SERVICE_RELEASE}")
-
-    # Define OpenTelemetry resources (e.g., service name)
-    resource = Resource(attributes={"service.name": OTEL_SERVICE_NAME})
-
-    # Set the tracer provider with the custom resource
-    trace.set_tracer_provider(TracerProvider(resource=resource))
-
-    # Configure the appropriate exporter based on the environment variable
-    if OTEL_EXPORTER_TYPE == "otlp":
-        logger.info("OpenTelemetry is using OTLP exporter")
-
-        exporter = OTLPSpanExporter(
-            endpoint=OTEL_OTLP_ENDPOINT,  # Default OTLP endpoint
-            insecure=True  # Disable TLS for local testing
-        )
-
-    else:
-        logger.info("OpenTelemetry is using Console exporter")
-
-        exporter = ConsoleSpanExporter()
-
-    # Replace the default span processor with the custom one
-    trace.get_tracer_provider().add_span_processor(GlobalTaggingSpanProcessor(exporter))