digitalkin 0.3.1.dev2__py3-none-any.whl → 0.3.2.dev14__py3-none-any.whl

monitoring/digitalkin_observability/interceptors.py

@@ -0,0 +1,176 @@
+"""gRPC interceptors for automatic metrics collection.
+
+This module provides gRPC server interceptors that automatically track
+request duration and errors. Requires grpcio package.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+    from typing import Any
+
+    import grpc
+
+from digitalkin_observability.metrics import get_metrics
+
+
+class MetricsServerInterceptor:
+    """Intercepts all gRPC calls to collect metrics.
+
+    This interceptor automatically tracks:
+    - Request duration (histogram)
+    - Error counts
+
+    Usage:
+        import grpc
+        from digitalkin_observability import MetricsServerInterceptor
+
+        interceptors = [MetricsServerInterceptor()]
+        server = grpc.aio.server(interceptors=interceptors)
+    """
+
+    async def intercept_service(
+        self,
+        continuation: Callable[["grpc.HandlerCallDetails"], Awaitable["grpc.RpcMethodHandler"]],
+        handler_call_details: "grpc.HandlerCallDetails",
+    ) -> "grpc.RpcMethodHandler":
+        """Intercept a gRPC service call to collect metrics.
+
+        Args:
+            continuation: The next interceptor or the actual handler.
+            handler_call_details: Details about the call being intercepted.
+
+        Returns:
+            The RPC method handler.
+        """
+        start = time.perf_counter()
+        metrics = get_metrics()
+
+        try:
+            handler = await continuation(handler_call_details)
+            return _MetricsWrappedHandler(handler, start, handler_call_details.method)
+        except Exception:
+            metrics.inc_errors()
+            metrics.observe_grpc_duration(time.perf_counter() - start)
+            raise
+
+
+class _MetricsWrappedHandler:
+    """Wrapper that measures actual handler execution time."""
+
+    def __init__(
+        self,
+        handler: "grpc.RpcMethodHandler",
+        start_time: float,
+        method: str,
+    ) -> None:
+        self._handler = handler
+        self._start_time = start_time
+        self._method = method
+
+        # Copy attributes from original handler
+        self.request_streaming = handler.request_streaming
+        self.response_streaming = handler.response_streaming
+        self.request_deserializer = handler.request_deserializer
+        self.response_serializer = handler.response_serializer
+
+        # Wrap the appropriate method based on streaming type
+        if handler.unary_unary:
+            self.unary_unary = self._wrap_unary_unary(handler.unary_unary)
+            self.unary_stream = None
+            self.stream_unary = None
+            self.stream_stream = None
+        elif handler.unary_stream:
+            self.unary_unary = None
+            self.unary_stream = self._wrap_unary_stream(handler.unary_stream)
+            self.stream_unary = None
+            self.stream_stream = None
+        elif handler.stream_unary:
+            self.unary_unary = None
+            self.unary_stream = None
+            self.stream_unary = self._wrap_stream_unary(handler.stream_unary)
+            self.stream_stream = None
+        elif handler.stream_stream:
+            self.unary_unary = None
+            self.unary_stream = None
+            self.stream_unary = None
+            self.stream_stream = self._wrap_stream_stream(handler.stream_stream)
+        else:
+            self.unary_unary = None
+            self.unary_stream = None
+            self.stream_unary = None
+            self.stream_stream = None
+
+    def _wrap_unary_unary(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]]:
+        """Wrap a unary-unary handler."""
+        async def wrapped(request: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                return await handler(request, context)
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped
+
+    def _wrap_unary_stream(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], "Any"],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], "Any"]:
+        """Wrap a unary-stream handler."""
+        async def wrapped(request: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                async for response in handler(request, context):
+                    yield response
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped
+
+    def _wrap_stream_unary(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]]:
+        """Wrap a stream-unary handler."""
+        async def wrapped(request_iterator: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                return await handler(request_iterator, context)
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped
+
+    def _wrap_stream_stream(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], "Any"],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], "Any"]:
+        """Wrap a stream-stream handler."""
+        async def wrapped(request_iterator: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                async for response in handler(request_iterator, context):
+                    yield response
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped

monitoring/digitalkin_observability/metrics.py

@@ -0,0 +1,201 @@
+"""Core metrics collection for DigitalKin.
+
+This module provides a thread-safe singleton MetricsCollector that tracks
+various metrics about job execution, gRPC requests, and system performance.
+
+No external dependencies required.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from threading import Lock
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+@dataclass
+class Histogram:
+    """Simple histogram with configurable buckets."""
+
+    buckets: tuple[float, ...] = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
+    counts: dict[float, int] = field(default_factory=lambda: defaultdict(int))
+    total_sum: float = 0.0
+    count: int = 0
+
+    def observe(self, value: float) -> None:
+        """Record an observation in the histogram."""
+        self.total_sum += value
+        self.count += 1
+        for bucket in self.buckets:
+            if value <= bucket:
+                self.counts[bucket] += 1
+
+    def reset(self) -> None:
+        """Reset histogram state."""
+        self.counts = defaultdict(int)
+        self.total_sum = 0.0
+        self.count = 0
+
+
+class MetricsCollector:
+    """Thread-safe singleton metrics collector.
+
+    Collects various metrics about job execution, gRPC requests,
+    and system performance. Designed to be stateless per-request
+    while maintaining aggregate counters.
+
+    Usage:
+        metrics = MetricsCollector()  # or get_metrics()
+        metrics.inc_jobs_started("my_module")
+        metrics.inc_jobs_completed("my_module", duration=1.5)
+        print(metrics.snapshot())
+    """
+
+    _instance: ClassVar[MetricsCollector | None] = None
+    _lock: ClassVar[Lock] = Lock()
+
+    def __new__(cls) -> "MetricsCollector":
+        """Create or return the singleton instance."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    instance = super().__new__(cls)
+                    instance._init_metrics()
+                    cls._instance = instance
+        return cls._instance
+
+    def _init_metrics(self) -> None:
+        """Initialize all metric storage."""
+        # Counters
+        self.jobs_started_total: int = 0
+        self.jobs_completed_total: int = 0
+        self.jobs_failed_total: int = 0
+        self.jobs_cancelled_total: int = 0
+        self.messages_sent_total: int = 0
+        self.heartbeats_sent_total: int = 0
+        self.errors_total: int = 0
+
+        # Gauges
+        self.active_jobs: int = 0
+        self.active_connections: int = 0
+        self.queue_depth: dict[str, int] = {}
+
+        # Histograms
+        self.job_duration_seconds = Histogram()
+        self.message_latency_seconds = Histogram()
+        self.grpc_request_duration_seconds = Histogram()
+
+        # Labels for breakdown
+        self._by_module: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+        self._by_protocol: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+
+        # Instance lock for thread safety
+        self._instance_lock = Lock()
+
+    def inc_jobs_started(self, module_name: str) -> None:
+        """Increment jobs started counter."""
+        with self._instance_lock:
+            self.jobs_started_total += 1
+            self.active_jobs += 1
+            self._by_module[module_name]["started"] += 1
+
+    def inc_jobs_completed(self, module_name: str, duration: float) -> None:
+        """Increment jobs completed counter and record duration."""
+        with self._instance_lock:
+            self.jobs_completed_total += 1
+            self.active_jobs = max(0, self.active_jobs - 1)
+            self._by_module[module_name]["completed"] += 1
+            self.job_duration_seconds.observe(duration)
+
+    def inc_jobs_failed(self, module_name: str) -> None:
+        """Increment jobs failed counter."""
+        with self._instance_lock:
+            self.jobs_failed_total += 1
+            self.active_jobs = max(0, self.active_jobs - 1)
+            self._by_module[module_name]["failed"] += 1
+
+    def inc_jobs_cancelled(self, module_name: str) -> None:
+        """Increment jobs cancelled counter."""
+        with self._instance_lock:
+            self.jobs_cancelled_total += 1
+            self.active_jobs = max(0, self.active_jobs - 1)
+            self._by_module[module_name]["cancelled"] += 1
+
+    def inc_messages_sent(self, protocol: str | None = None) -> None:
+        """Increment messages sent counter."""
+        with self._instance_lock:
+            self.messages_sent_total += 1
+            if protocol:
+                self._by_protocol[protocol]["messages"] += 1
+
+    def inc_heartbeats_sent(self) -> None:
+        """Increment heartbeats sent counter."""
+        with self._instance_lock:
+            self.heartbeats_sent_total += 1
+
+    def inc_errors(self) -> None:
+        """Increment errors counter."""
+        with self._instance_lock:
+            self.errors_total += 1
+
+    def set_queue_depth(self, job_id: str, depth: int) -> None:
+        """Set the queue depth for a job."""
+        with self._instance_lock:
+            self.queue_depth[job_id] = depth
+
+    def clear_queue_depth(self, job_id: str) -> None:
+        """Clear queue depth tracking for a job."""
+        with self._instance_lock:
+            self.queue_depth.pop(job_id, None)
+
+    def observe_grpc_duration(self, duration: float) -> None:
+        """Record a gRPC request duration."""
+        with self._instance_lock:
+            self.grpc_request_duration_seconds.observe(duration)
+
+    def observe_message_latency(self, latency: float) -> None:
+        """Record a message latency."""
+        with self._instance_lock:
+            self.message_latency_seconds.observe(latency)
+
+    def snapshot(self) -> dict[str, Any]:
+        """Return current metrics as dict for export."""
+        with self._instance_lock:
+            return {
+                "jobs_started_total": self.jobs_started_total,
+                "jobs_completed_total": self.jobs_completed_total,
+                "jobs_failed_total": self.jobs_failed_total,
+                "jobs_cancelled_total": self.jobs_cancelled_total,
+                "active_jobs": self.active_jobs,
+                "messages_sent_total": self.messages_sent_total,
+                "heartbeats_sent_total": self.heartbeats_sent_total,
+                "errors_total": self.errors_total,
+                "active_connections": self.active_connections,
+                "total_queue_depth": sum(self.queue_depth.values()),
+                "job_duration_seconds": {
+                    "count": self.job_duration_seconds.count,
+                    "sum": self.job_duration_seconds.total_sum,
+                    "buckets": dict(self.job_duration_seconds.counts),
+                },
+                "grpc_request_duration_seconds": {
+                    "count": self.grpc_request_duration_seconds.count,
+                    "sum": self.grpc_request_duration_seconds.total_sum,
+                    "buckets": dict(self.grpc_request_duration_seconds.counts),
+                },
+                "by_module": {k: dict(v) for k, v in self._by_module.items()},
+                "by_protocol": {k: dict(v) for k, v in self._by_protocol.items()},
+            }
+
+    def reset(self) -> None:
+        """Reset all metrics. Useful for testing."""
+        with self._instance_lock:
+            self._init_metrics()
+
+
+def get_metrics() -> MetricsCollector:
+    """Get the global MetricsCollector instance."""
+    return MetricsCollector()

monitoring/digitalkin_observability/prometheus.py

@@ -0,0 +1,137 @@
+"""Prometheus metrics exporter for DigitalKin.
+
+This module exports metrics in Prometheus text exposition format.
+No external dependencies required.
+"""
+
+from __future__ import annotations
+
+from digitalkin_observability.metrics import get_metrics
+
+
+class PrometheusExporter:
+    """Exports metrics in Prometheus text format.
+
+    Usage:
+        output = PrometheusExporter.export()
+        # Returns Prometheus-compatible text format
+    """
+
+    @staticmethod
+    def export() -> str:
+        """Generate Prometheus-compatible metrics output."""
+        snapshot = get_metrics().snapshot()
+        lines: list[str] = []
+
+        # Counters
+        lines.extend([
+            "# HELP digitalkin_jobs_started_total Total jobs started",
+            "# TYPE digitalkin_jobs_started_total counter",
+            f"digitalkin_jobs_started_total {snapshot['jobs_started_total']}",
+            "",
+            "# HELP digitalkin_jobs_completed_total Total jobs completed successfully",
+            "# TYPE digitalkin_jobs_completed_total counter",
+            f"digitalkin_jobs_completed_total {snapshot['jobs_completed_total']}",
+            "",
+            "# HELP digitalkin_jobs_failed_total Total jobs failed",
+            "# TYPE digitalkin_jobs_failed_total counter",
+            f"digitalkin_jobs_failed_total {snapshot['jobs_failed_total']}",
+            "",
+            "# HELP digitalkin_jobs_cancelled_total Total jobs cancelled",
+            "# TYPE digitalkin_jobs_cancelled_total counter",
+            f"digitalkin_jobs_cancelled_total {snapshot['jobs_cancelled_total']}",
+            "",
+            "# HELP digitalkin_messages_sent_total Total messages sent",
+            "# TYPE digitalkin_messages_sent_total counter",
+            f"digitalkin_messages_sent_total {snapshot['messages_sent_total']}",
+            "",
+            "# HELP digitalkin_heartbeats_sent_total Total heartbeats sent",
+            "# TYPE digitalkin_heartbeats_sent_total counter",
+            f"digitalkin_heartbeats_sent_total {snapshot['heartbeats_sent_total']}",
+            "",
+            "# HELP digitalkin_errors_total Total errors",
+            "# TYPE digitalkin_errors_total counter",
+            f"digitalkin_errors_total {snapshot['errors_total']}",
+            "",
+        ])
+
+        # Gauges
+        lines.extend([
+            "# HELP digitalkin_active_jobs Current number of active jobs",
+            "# TYPE digitalkin_active_jobs gauge",
+            f"digitalkin_active_jobs {snapshot['active_jobs']}",
+            "",
+            "# HELP digitalkin_active_connections Current number of active connections",
+            "# TYPE digitalkin_active_connections gauge",
+            f"digitalkin_active_connections {snapshot['active_connections']}",
+            "",
+            "# HELP digitalkin_total_queue_depth Total items in all job queues",
+            "# TYPE digitalkin_total_queue_depth gauge",
+            f"digitalkin_total_queue_depth {snapshot['total_queue_depth']}",
+            "",
+        ])
+
+        # Job duration histogram
+        lines.extend(PrometheusExporter._format_histogram(
+            "digitalkin_job_duration_seconds",
+            "Job execution duration in seconds",
+            snapshot["job_duration_seconds"],
+        ))
+
+        # gRPC request duration histogram
+        lines.extend(PrometheusExporter._format_histogram(
+            "digitalkin_grpc_request_duration_seconds",
+            "gRPC request duration in seconds",
+            snapshot["grpc_request_duration_seconds"],
+        ))
+
+        # Per-module breakdown
+        if snapshot["by_module"]:
+            lines.extend([
+                "",
+                "# HELP digitalkin_jobs_by_module Jobs breakdown by module and status",
+                "# TYPE digitalkin_jobs_by_module counter",
+            ])
+            for module_name, counts in snapshot["by_module"].items():
+                for status, value in counts.items():
+                    lines.append(
+                        f'digitalkin_jobs_by_module{{module="{module_name}",status="{status}"}} {value}'
+                    )
+
+        # Per-protocol breakdown
+        if snapshot["by_protocol"]:
+            lines.extend([
+                "",
+                "# HELP digitalkin_messages_by_protocol Messages breakdown by protocol",
+                "# TYPE digitalkin_messages_by_protocol counter",
+            ])
+            for protocol, counts in snapshot["by_protocol"].items():
+                for metric, value in counts.items():
+                    lines.append(
+                        f'digitalkin_messages_by_protocol{{protocol="{protocol}",metric="{metric}"}} {value}'
+                    )
+
+        return "\n".join(lines)
+
+    @staticmethod
+    def _format_histogram(name: str, help_text: str, data: dict) -> list[str]:
+        """Format a histogram for Prometheus output."""
+        lines = [
+            "",
+            f"# HELP {name} {help_text}",
+            f"# TYPE {name} histogram",
+        ]
+
+        # Sort buckets and output cumulative counts
+        cumulative = 0
+        for bucket in sorted(data.get("buckets", {}).keys()):
+            cumulative += data["buckets"][bucket]
+            lines.append(f'{name}_bucket{{le="{bucket}"}} {cumulative}')
+
+        lines.extend([
+            f'{name}_bucket{{le="+Inf"}} {data.get("count", 0)}',
+            f'{name}_sum {data.get("sum", 0)}',
+            f'{name}_count {data.get("count", 0)}',
+        ])
+
+        return lines

monitoring/tests/test_metrics.py

@@ -0,0 +1,172 @@
+"""Tests for metrics collection.
+
+Run with: python -m pytest tests/test_metrics.py
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+# Add the parent directory to the path so we can import digitalkin_observability
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from digitalkin_observability import MetricsCollector, PrometheusExporter, get_metrics
+
+
+class TestMetricsCollector:
+    """Tests for MetricsCollector singleton."""
+
+    def setup_method(self) -> None:
+        """Reset metrics before each test."""
+        get_metrics().reset()
+
+    def test_singleton_returns_same_instance(self) -> None:
+        """Test that get_metrics returns the same instance."""
+        m1 = get_metrics()
+        m2 = get_metrics()
+        assert m1 is m2
+
+    def test_inc_jobs_started(self) -> None:
+        """Test incrementing jobs started counter."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("TestModule")
+
+        assert metrics.jobs_started_total == 1
+        assert metrics.active_jobs == 1
+
+    def test_inc_jobs_completed(self) -> None:
+        """Test incrementing jobs completed counter."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("TestModule")
+        metrics.inc_jobs_completed("TestModule", 1.5)
+
+        assert metrics.jobs_completed_total == 1
+        assert metrics.active_jobs == 0
+        assert metrics.job_duration_seconds.count == 1
+        assert metrics.job_duration_seconds.total_sum == 1.5
+
+    def test_inc_jobs_failed(self) -> None:
+        """Test incrementing jobs failed counter."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("TestModule")
+        metrics.inc_jobs_failed("TestModule")
+
+        assert metrics.jobs_failed_total == 1
+        assert metrics.active_jobs == 0
+
+    def test_inc_jobs_cancelled(self) -> None:
+        """Test incrementing jobs cancelled counter."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("TestModule")
+        metrics.inc_jobs_cancelled("TestModule")
+
+        assert metrics.jobs_cancelled_total == 1
+        assert metrics.active_jobs == 0
+
+    def test_inc_messages_sent(self) -> None:
+        """Test incrementing messages sent counter."""
+        metrics = get_metrics()
+        metrics.inc_messages_sent("message")
+        metrics.inc_messages_sent("file")
+        metrics.inc_messages_sent()
+
+        assert metrics.messages_sent_total == 3
+
+    def test_queue_depth_tracking(self) -> None:
+        """Test queue depth tracking."""
+        metrics = get_metrics()
+        metrics.set_queue_depth("job1", 5)
+        metrics.set_queue_depth("job2", 3)
+
+        assert metrics.queue_depth["job1"] == 5
+        assert metrics.queue_depth["job2"] == 3
+
+        metrics.clear_queue_depth("job1")
+        assert "job1" not in metrics.queue_depth
+
+    def test_snapshot(self) -> None:
+        """Test snapshot returns all metrics."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("TestModule")
+        metrics.inc_jobs_completed("TestModule", 0.5)
+        metrics.inc_messages_sent("message")
+
+        snapshot = metrics.snapshot()
+
+        assert snapshot["jobs_started_total"] == 1
+        assert snapshot["jobs_completed_total"] == 1
+        assert snapshot["messages_sent_total"] == 1
+        assert "job_duration_seconds" in snapshot
+        assert "by_module" in snapshot
+        assert "TestModule" in snapshot["by_module"]
+
+    def test_histogram_observe(self) -> None:
+        """Test histogram observations."""
+        metrics = get_metrics()
+        metrics.observe_grpc_duration(0.05)
+        metrics.observe_grpc_duration(0.15)
+
+        assert metrics.grpc_request_duration_seconds.count == 2
+        assert metrics.grpc_request_duration_seconds.total_sum == pytest.approx(0.2)
+
+    def test_reset_clears_all_metrics(self) -> None:
+        """Test reset clears all metrics."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("TestModule")
+        metrics.inc_errors()
+
+        metrics.reset()
+
+        assert metrics.jobs_started_total == 0
+        assert metrics.errors_total == 0
+        assert metrics.active_jobs == 0
+
+
+class TestPrometheusExporter:
+    """Tests for Prometheus exporter."""
+
+    def setup_method(self) -> None:
+        """Reset metrics before each test."""
+        get_metrics().reset()
+
+    def test_export_returns_string(self) -> None:
+        """Test that export returns a string."""
+        output = PrometheusExporter.export()
+        assert isinstance(output, str)
+
+    def test_export_contains_job_counters(self) -> None:
+        """Test export contains job counters."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("TestModule")
+
+        output = PrometheusExporter.export()
+
+        assert "digitalkin_jobs_started_total 1" in output
+        assert "digitalkin_active_jobs 1" in output
+
+    def test_export_contains_histogram(self) -> None:
+        """Test export contains histogram data."""
+        metrics = get_metrics()
+        metrics.observe_grpc_duration(0.05)
+
+        output = PrometheusExporter.export()
+
+        assert "digitalkin_grpc_request_duration_seconds" in output
+        assert "# TYPE digitalkin_grpc_request_duration_seconds histogram" in output
+
+    def test_export_contains_module_breakdown(self) -> None:
+        """Test export contains per-module breakdown."""
+        metrics = get_metrics()
+        metrics.inc_jobs_started("MyModule")
+
+        output = PrometheusExporter.export()
+
+        assert 'digitalkin_jobs_by_module{module="MyModule",status="started"} 1' in output
+
+    def test_export_contains_help_and_type(self) -> None:
+        """Test export contains HELP and TYPE comments."""
+        output = PrometheusExporter.export()
+
+        assert "# HELP digitalkin_jobs_started_total" in output
+        assert "# TYPE digitalkin_jobs_started_total counter" in output