genkit-plugin-google-cloud 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

genkit/plugins/google_cloud/telemetry/metrics.py

@@ -0,0 +1,246 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""AI monitoring metrics for Genkit.
+
+This module provides lazy-initialized OpenTelemetry metrics for AI operations.
+Metrics are exported to Google Cloud Monitoring with the workload.googleapis.com
+prefix by default.
+
+Metrics Defined:
+    Input metrics:
+        - genkit/ai/generate/input/tokens
+        - genkit/ai/generate/input/characters
+        - genkit/ai/generate/input/images
+        - genkit/ai/generate/input/videos
+        - genkit/ai/generate/input/audio
+
+    Output metrics:
+        - genkit/ai/generate/output/tokens
+        - genkit/ai/generate/output/characters
+        - genkit/ai/generate/output/images
+        - genkit/ai/generate/output/videos
+        - genkit/ai/generate/output/audio
+
+    Thinking metrics:
+        - genkit/ai/generate/thinking/tokens
+
+See Also:
+    - Cloud Monitoring Custom Metrics: https://cloud.google.com/monitoring/custom-metrics
+    - Workload Metrics: https://cloud.google.com/monitoring/api/metrics_other
+"""
+
+import contextlib
+import json
+import re
+
+import structlog
+from opentelemetry import metrics
+from opentelemetry.sdk.trace import ReadableSpan
+
+logger = structlog.get_logger(__name__)
+
+meter = metrics.get_meter('genkit')
+
+
+def _metric(name: str, desc: str, unit: str = '1') -> tuple[str, str, str]:
+    """Create metric name with genkit/ai/ prefix.
+
+    Args:
+        name: Metric name
+        desc: Metric description
+        unit: Metric unit (default: '1')
+
+    Returns:
+        Tuple of (prefixed_name, description, unit)
+    """
+    return f'genkit/ai/{name}', desc, unit
+
+
+# Metric caches for lazy initialization
+_counter_cache: dict[str, metrics.Counter] = {}
+_histogram_cache: dict[str, metrics.Histogram] = {}
+
+
+def _get_counter(name: str, desc: str, unit: str = '1') -> metrics.Counter:
+    """Get or create counter metric with lazy initialization.
+
+    Args:
+        name: Metric name
+        desc: Metric description
+        unit: Metric unit (default: '1')
+
+    Returns:
+        OpenTelemetry Counter metric
+    """
+    if name not in _counter_cache:
+        _counter_cache[name] = meter.create_counter(name, description=desc, unit=unit)
+    return _counter_cache[name]
+
+
+def _get_histogram(name: str, desc: str, unit: str = '1') -> metrics.Histogram:
+    """Get or create histogram metric with lazy initialization.
+
+    Args:
+        name: Metric name
+        desc: Metric description
+        unit: Metric unit (default: '1')
+
+    Returns:
+        OpenTelemetry Histogram metric
+    """
+    if name not in _histogram_cache:
+        _histogram_cache[name] = meter.create_histogram(name, description=desc, unit=unit)
+    return _histogram_cache[name]
+
+
+# Metric definitions
+def _requests() -> metrics.Counter:
+    return _get_counter(*_metric('generate/requests', 'Generate requests'))
+
+
+def _failures() -> metrics.Counter:
+    return _get_counter(*_metric('generate/failures', 'Generate failures'))
+
+
+def _latency() -> metrics.Histogram:
+    return _get_histogram(*_metric('generate/latency', 'Generate latency', 'ms'))
+
+
+def _input_tokens() -> metrics.Counter:
+    return _get_counter(*_metric('generate/input/tokens', 'Input tokens'))
+
+
+def _output_tokens() -> metrics.Counter:
+    return _get_counter(*_metric('generate/output/tokens', 'Output tokens'))
+
+
+def _input_characters() -> metrics.Counter:
+    return _get_counter(*_metric('generate/input/characters', 'Input characters'))
+
+
+def _output_characters() -> metrics.Counter:
+    return _get_counter(*_metric('generate/output/characters', 'Output characters'))
+
+
+def _input_images() -> metrics.Counter:
+    return _get_counter(*_metric('generate/input/images', 'Input images'))
+
+
+def _output_images() -> metrics.Counter:
+    return _get_counter(*_metric('generate/output/images', 'Output images'))
+
+
+def _input_videos() -> metrics.Counter:
+    return _get_counter(*_metric('generate/input/videos', 'Input videos'))
+
+
+def _output_videos() -> metrics.Counter:
+    return _get_counter(*_metric('generate/output/videos', 'Output videos'))
+
+
+def _input_audio() -> metrics.Counter:
+    return _get_counter(*_metric('generate/input/audio', 'Input audio'))
+
+
+def _output_audio() -> metrics.Counter:
+    return _get_counter(*_metric('generate/output/audio', 'Output audio'))
+
+
+def record_generate_metrics(span: ReadableSpan) -> None:
+    """Record AI monitoring metrics from a model action span.
+
+    Args:
+        span: OpenTelemetry span containing model execution data
+    """
+    attrs = span.attributes
+    if not attrs:
+        return
+
+    # Check if this is a model action
+    if attrs.get('genkit:type') != 'action' or attrs.get('genkit:metadata:subtype') != 'model':
+        return
+
+    # Extract dimensions
+    model = str(attrs.get('genkit:name', '<unknown>'))[:1000]
+    path = str(attrs.get('genkit:path', ''))[:1000]
+    source = _extract_feature_name(path)
+    is_error = not span.status.is_ok
+    error = 'error' if is_error else 'none'
+
+    dimensions = {'model': model, 'source': source, 'error': error}
+
+    try:
+        _requests().add(1, dimensions)
+        if is_error:
+            _failures().add(1, dimensions)
+
+        # Latency
+        latency_ms = None
+        if span.end_time and span.start_time:
+            latency_ms = (span.end_time - span.start_time) / 1_000_000
+            _latency().record(latency_ms, dimensions)
+
+        usage = {}
+        output_json = attrs.get('genkit:output')
+        if output_json and isinstance(output_json, str):
+            try:
+                output_data = json.loads(output_json)
+                usage = output_data.get('usage', {})
+            except (json.JSONDecodeError, AttributeError):
+                pass
+
+        usage_metrics = {
+            'inputTokens': _input_tokens,
+            'outputTokens': _output_tokens,
+            'inputCharacters': _input_characters,
+            'outputCharacters': _output_characters,
+            'inputImages': _input_images,
+            'outputImages': _output_images,
+            'inputVideos': _input_videos,
+            'outputVideos': _output_videos,
+            'inputAudio': _input_audio,
+            'outputAudio': _output_audio,
+        }
+
+        for key, metric_fn in usage_metrics.items():
+            value = usage.get(key)
+            if value is not None:
+                with contextlib.suppress(ValueError, TypeError):
+                    metric_fn().add(int(value), dimensions)
+
+    except Exception as e:
+        logger.warning('Error recording metrics', error=str(e))
+
+
+def _extract_feature_name(path: str) -> str:
+    """Extract feature name from Genkit action path.
+
+    Args:
+        path: Genkit action path in format '/{name,t:type}' or '/{outer,t:flow}/{inner,t:flow}'
+
+    Returns:
+        Extracted feature name or '<unknown>' if path cannot be parsed
+    """
+    if not path:
+        return '<unknown>'
+
+    parts = path.split('/')
+    if len(parts) < 2:
+        return '<unknown>'
+
+    match = re.match(r'\{([^,}]+)', parts[1])
+    return match.group(1) if match else '<unknown>'

genkit/plugins/google_cloud/telemetry/path.py

@@ -0,0 +1,157 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Path telemetry for GCP.
+
+This module tracks path-level failure metrics and logs errors,
+matching the JavaScript implementation.
+
+Metrics Recorded:
+    - genkit/feature/path/requests: Counter for unique flow paths
+    - genkit/feature/path/latency: Histogram for path latency (ms)
+
+Cross-Language Parity:
+    - JavaScript: js/plugins/google-cloud/src/telemetry/paths.ts
+    - Go: go/plugins/googlecloud/paths.go
+
+See Also:
+    - Cloud Monitoring Custom Metrics: https://cloud.google.com/monitoring/custom-metrics
+"""
+
+from __future__ import annotations
+
+import structlog
+from opentelemetry import metrics
+from opentelemetry.sdk.trace import ReadableSpan
+
+from genkit.core import GENKIT_VERSION
+
+from .utils import (
+    create_common_log_attributes,
+    extract_error_message,
+    extract_error_name,
+    extract_error_stack,
+    extract_outer_feature_name_from_path,
+    to_display_path,
+    truncate_path,
+)
+
+logger = structlog.get_logger(__name__)
+
+# Lazy-initialized metrics
+_path_counter: metrics.Counter | None = None
+_path_latency: metrics.Histogram | None = None
+
+
+def _get_path_counter() -> metrics.Counter:
+    """Get or create the path requests counter."""
+    global _path_counter
+    if _path_counter is None:
+        meter = metrics.get_meter('genkit')
+        _path_counter = meter.create_counter(
+            'genkit/feature/path/requests',
+            description='Tracks unique flow paths per flow.',
+            unit='1',
+        )
+    return _path_counter
+
+
+def _get_path_latency() -> metrics.Histogram:
+    """Get or create the path latency histogram."""
+    global _path_latency
+    if _path_latency is None:
+        meter = metrics.get_meter('genkit')
+        _path_latency = meter.create_histogram(
+            'genkit/feature/path/latency',
+            description='Latencies per flow path.',
+            unit='ms',
+        )
+    return _path_latency
+
+
+class PathsTelemetry:
+    """Telemetry handler for Genkit paths (error tracking)."""
+
+    def tick(
+        self,
+        span: ReadableSpan,
+        log_input_and_output: bool,
+        project_id: str | None = None,
+    ) -> None:
+        """Record telemetry for a path span.
+
+        Only ticks metrics for failing, leaf spans (isFailureSource).
+
+        Args:
+            span: The span to record telemetry for.
+            log_input_and_output: Whether to log input/output (unused here).
+            project_id: Optional GCP project ID.
+        """
+        attrs = span.attributes or {}
+
+        path = str(attrs.get('genkit:path', ''))
+        is_failure_source = bool(attrs.get('genkit:isFailureSource'))
+        state = str(attrs.get('genkit:state', ''))
+
+        # Only tick metrics for failing, leaf spans
+        if not path or not is_failure_source or state != 'error':
+            return
+
+        session_id = str(attrs.get('genkit:sessionId', '')) or None
+        thread_name = str(attrs.get('genkit:threadName', '')) or None
+
+        events = list(span.events)
+        error_name = extract_error_name(events) or '<unknown>'
+        error_message = extract_error_message(events) or '<unknown>'
+        error_stack = extract_error_stack(events) or ''
+
+        # Calculate latency
+        latency_ms = 0.0
+        if span.end_time and span.start_time:
+            latency_ms = (span.end_time - span.start_time) / 1_000_000
+
+        path_dimensions = {
+            'featureName': extract_outer_feature_name_from_path(path)[:256],
+            'status': 'failure',
+            'error': error_name[:256],
+            'path': path[:256],
+            'source': 'py',
+            'sourceVersion': GENKIT_VERSION,
+        }
+        _get_path_counter().add(1, path_dimensions)
+        _get_path_latency().record(latency_ms, path_dimensions)
+
+        display_path = truncate_path(to_display_path(path))
+        log_attrs = {
+            **create_common_log_attributes(span, project_id),
+            'path': display_path,
+            'qualifiedPath': path,
+            'name': error_name,
+            'message': error_message,
+            'stack': error_stack,
+            'source': 'py',
+            'sourceVersion': GENKIT_VERSION,
+        }
+        if session_id:
+            log_attrs['sessionId'] = session_id
+        if thread_name:
+            log_attrs['threadName'] = thread_name
+
+        logger.error(f'Error[{display_path}, {error_name}]', **log_attrs)
+
+
+# Singleton instance
+paths_telemetry = PathsTelemetry()