grpcio-observability 1.71.0__cp311-cp311-musllinux_1_1_aarch64.whl

grpc_observability/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2023 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from grpc_observability._open_telemetry_plugin import OpenTelemetryPlugin
+
+__all__ = ("OpenTelemetryPlugin",)

grpc_observability/_measures.py

@@ -0,0 +1,91 @@
+# Copyright 2023 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from opencensus.stats import measure
+
+# These measure definitions should be kept in sync across opencensus implementations.
+# https://github.com/census-instrumentation/opencensus-java/blob/master/contrib/grpc_metrics/src/main/java/io/opencensus/contrib/grpc/metrics/RpcMeasureConstants.java.
+
+# Unit constants
+UNIT_BYTES = "By"
+UNIT_MILLISECONDS = "ms"
+UNIT_COUNT = "1"
+
+# Client
+CLIENT_STARTED_RPCS_MEASURE = measure.MeasureInt(
+    "grpc.io/client/started_rpcs",
+    "The total number of client RPCs ever opened, including those that have not been completed.",
+    UNIT_COUNT,
+)
+
+CLIENT_COMPLETED_RPCS_MEASURE = measure.MeasureInt(
+    "grpc.io/client/completed_rpcs",
+    "The total number of completed client RPCs",
+    UNIT_COUNT,
+)
+
+CLIENT_ROUNDTRIP_LATENCY_MEASURE = measure.MeasureFloat(
+    "grpc.io/client/roundtrip_latency",
+    "Time between first byte of request sent to last byte of response received, or terminal error",
+    UNIT_MILLISECONDS,
+)
+
+CLIENT_API_LATENCY_MEASURE = measure.MeasureInt(
+    "grpc.io/client/api_latency",
+    "End-to-end time taken to complete an RPC",
+    UNIT_MILLISECONDS,
+)
+
+CLIENT_SEND_BYTES_PER_RPC_MEASURE = measure.MeasureFloat(
+    "grpc.io/client/sent_bytes_per_rpc",
+    "Total bytes sent across all request messages per RPC",
+    UNIT_BYTES,
+)
+
+CLIENT_RECEIVED_BYTES_PER_RPC_MEASURE = measure.MeasureFloat(
+    "grpc.io/client/received_bytes_per_rpc",
+    "Total bytes received across all response messages per RPC",
+    UNIT_BYTES,
+)
+
+# Server
+SERVER_STARTED_RPCS_MEASURE = measure.MeasureInt(
+    "grpc.io/server/started_rpcs",
+    "Total bytes sent across all request messages per RPC",
+    UNIT_COUNT,
+)
+
+SERVER_COMPLETED_RPCS_MEASURE = measure.MeasureInt(
+    "grpc.io/server/completed_rpcs",
+    "The total number of completed server RPCs",
+    UNIT_COUNT,
+)
+
+SERVER_SENT_BYTES_PER_RPC_MEASURE = measure.MeasureFloat(
+    "grpc.io/server/sent_bytes_per_rpc",
+    "Total bytes sent across all messages per RPC",
+    UNIT_BYTES,
+)
+
+SERVER_RECEIVED_BYTES_PER_RPC_MEASURE = measure.MeasureFloat(
+    "grpc.io/server/received_bytes_per_rpc",
+    "Total bytes received across all messages per RPC",
+    UNIT_BYTES,
+)
+
+SERVER_SERVER_LATENCY_MEASURE = measure.MeasureFloat(
+    "grpc.io/server/server_latency",
+    "Time between first byte of request received to last byte of response sent, or terminal error",
+    UNIT_MILLISECONDS,
+)

grpc_observability/_observability.py

@@ -0,0 +1,119 @@
+# Copyright 2023 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import abc
+from dataclasses import dataclass
+from dataclasses import field
+import enum
+from typing import AnyStr, Dict, List, Mapping, Set, Tuple
+
+
+class Exporter(metaclass=abc.ABCMeta):
+    """Abstract base class for census data exporters."""
+
+    @abc.abstractmethod
+    def export_stats_data(self, stats_data: List[TracingData]) -> None:
+        """Exports a list of TracingData objects to the exporter's destination.
+
+        Args:
+          stats_data: A list of TracingData objects to export.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def export_tracing_data(self, tracing_data: List[StatsData]) -> None:
+        """Exports a list of StatsData objects to the exporter's destination.
+
+        Args:
+          tracing_data: A list of StatsData objects to export.
+        """
+        raise NotImplementedError()
+
+
+@dataclass(frozen=True)
+class StatsData:
+    """A data class representing stats data.
+
+    Attributes:
+      name: An element of grpc_observability._cyobservability.MetricsName, e.g.
+        MetricsName.CLIENT_STARTED_RPCS.
+      measure_double: A bool indicate whether the metric is a floating-point
+        value.
+      value_int: The actual metric value if measure_double is False.
+      value_float: The actual metric value if measure_double is True.
+      include_exchange_labels: Whether this data should include exchanged labels.
+      labels: A dictionary that maps label tags associated with this metric to
+       corresponding label value.
+      identifiers: A set of strings identifying which stats plugins this StatsData
+        belongs to.
+      registered_method: Whether the method in this data is a registered method
+        in stubs.
+    """
+
+    name: "grpc_observability._cyobservability.MetricsName"
+    measure_double: bool
+    value_int: int = 0
+    value_float: float = 0.0
+    include_exchange_labels: bool = False
+    labels: Dict[str, AnyStr] = field(default_factory=dict)
+    identifiers: Set[str] = field(default_factory=set)
+    registered_method: bool = False
+
+
+@dataclass(frozen=True)
+class TracingData:
+    """A data class representing tracing data.
+
+    Attributes:
+      name: The name for tracing data, also the name for the Span.
+      start_time: The start time for the span in RFC3339 UTC "Zulu" format, e.g.
+       2014-10-02T15:01:23Z
+      end_time: The end time for the span in RFC3339 UTC "Zulu" format, e.g.
+       2014-10-02T15:01:23Z
+      trace_id: The identifier for the trace associated with this span as a
+       32-character hexadecimal encoded string,
+       e.g. 26ed0036f2eff2b7317bccce3e28d01f
+      span_id: The identifier for the span as a 16-character hexadecimal encoded
+       string. e.g. 113ec879e62583bc
+      parent_span_id: An option identifier for the span's parent id.
+      status: An element of grpc.StatusCode in string format representing the
+       final status for the trace data.
+      should_sample: A bool indicates whether the span is sampled.
+      child_span_count: The number of child span associated with this span.
+      span_labels: A dictionary that maps labels tags associated with this
+       span to corresponding label value.
+      span_annotations: A dictionary that maps annotation timeStamp with
+       description. The timeStamp have a format which can be converted
+       to Python datetime.datetime, e.g. 2023-05-29 17:07:09.895
+    """
+
+    name: str
+    start_time: str
+    end_time: str
+    trace_id: str
+    span_id: str
+    parent_span_id: str
+    status: str
+    should_sample: bool
+    child_span_count: int
+    span_labels: Mapping[str, AnyStr] = field(default_factory=dict)
+    span_annotations: List[Tuple[str, str]] = field(default_factory=list)
+
+
+@enum.unique
+class OptionalLabelType(enum.Enum):
+    """What kinds of optional labels to add to metrics."""
+
+    XDS_SERVICE_LABELS = "kXdsServiceLabels"

grpc_observability/_observability_config.py

@@ -0,0 +1,129 @@
+# Copyright 2023 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helper to read observability config."""
+
+from dataclasses import dataclass
+from dataclasses import field
+import json
+import os
+from typing import Mapping, Optional
+
+GRPC_GCP_OBSERVABILITY_CONFIG_FILE_ENV = "GRPC_GCP_OBSERVABILITY_CONFIG_FILE"
+GRPC_GCP_OBSERVABILITY_CONFIG_ENV = "GRPC_GCP_OBSERVABILITY_CONFIG"
+
+
+@dataclass
+class GcpObservabilityConfig:
+    project_id: str = ""
+    stats_enabled: bool = False
+    tracing_enabled: bool = False
+    labels: Optional[Mapping[str, str]] = field(default_factory=dict)
+    sampling_rate: Optional[float] = 0.0
+
+    def load_from_string_content(self, config_contents: str) -> None:
+        """Loads the configuration from a string.
+
+        Args:
+            config_contents: The configuration string.
+
+        Raises:
+            ValueError: If the configuration is invalid.
+        """
+        try:
+            config_json = json.loads(config_contents)
+        except json.decoder.JSONDecodeError:
+            raise ValueError("Failed to load Json configuration.")
+
+        if config_json and not isinstance(config_json, dict):
+            raise ValueError("Found invalid configuration.")
+
+        self.project_id = config_json.get("project_id", "")
+        self.labels = config_json.get("labels", {})
+        self.stats_enabled = "cloud_monitoring" in config_json.keys()
+        self.tracing_enabled = "cloud_trace" in config_json.keys()
+        tracing_config = config_json.get("cloud_trace", {})
+        self.sampling_rate = tracing_config.get("sampling_rate", 0.0)
+
+
+def read_config() -> GcpObservabilityConfig:
+    """Reads the GCP observability config from the environment variables.
+
+    Returns:
+        The GCP observability config.
+
+    Raises:
+        ValueError: If the configuration is invalid.
+    """
+    config_contents = _get_gcp_observability_config_contents()
+    config = GcpObservabilityConfig()
+    config.load_from_string_content(config_contents)
+
+    if not config.project_id:
+        # Get project ID from GCP environment variables since project ID was not
+        # set it in the GCP observability config.
+        config.project_id = _get_gcp_project_id_from_env_var()
+        if not config.project_id:
+            # Could not find project ID from GCP environment variables either.
+            raise ValueError("GCP Project ID not found.")
+    return config
+
+
+def _get_gcp_project_id_from_env_var() -> Optional[str]:
+    """Gets the project ID from the GCP environment variables.
+
+    Returns:
+        The project ID, or an empty string if the project ID could not be found.
+    """
+
+    project_id = ""
+    project_id = os.getenv("GCP_PROJECT")
+    if project_id:
+        return project_id
+
+    project_id = os.getenv("GCLOUD_PROJECT")
+    if project_id:
+        return project_id
+
+    project_id = os.getenv("GOOGLE_CLOUD_PROJECT")
+    if project_id:
+        return project_id
+
+    return project_id
+
+
+def _get_gcp_observability_config_contents() -> str:
+    """Get the contents of the observability config from environment variable or file.
+
+    Returns:
+        The content from environment variable.
+
+    Raises:
+        ValueError: If no configuration content was found.
+    """
+
+    contents_str = ""
+    # First try get config from GRPC_GCP_OBSERVABILITY_CONFIG_FILE_ENV.
+    config_path = os.getenv(GRPC_GCP_OBSERVABILITY_CONFIG_FILE_ENV)
+    if config_path:
+        with open(config_path, "r") as f:
+            contents_str = f.read()
+
+    # Next, try GRPC_GCP_OBSERVABILITY_CONFIG_ENV env var.
+    if not contents_str:
+        contents_str = os.getenv(GRPC_GCP_OBSERVABILITY_CONFIG_ENV)
+
+    if not contents_str:
+        raise ValueError("Configuration content not found.")
+
+    return contents_str

grpc_observability/_open_census_exporter.py

@@ -0,0 +1,301 @@
+# Copyright 2023 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+import os
+from typing import List, Mapping, Optional, Tuple
+
+from google.rpc import code_pb2
+from grpc_observability import _observability  # pytype: disable=pyi-error
+from grpc_observability import _observability_config
+from grpc_observability import _views
+from opencensus.common.transports import async_
+from opencensus.ext.stackdriver import stats_exporter
+from opencensus.ext.stackdriver import trace_exporter
+from opencensus.stats import stats as stats_module
+from opencensus.stats.stats_recorder import StatsRecorder
+from opencensus.stats.view_manager import ViewManager
+from opencensus.tags.tag_key import TagKey
+from opencensus.tags.tag_map import TagMap
+from opencensus.tags.tag_value import TagValue
+from opencensus.trace import execution_context
+from opencensus.trace import samplers
+from opencensus.trace import span
+from opencensus.trace import span_context as span_context_module
+from opencensus.trace import span_data as span_data_module
+from opencensus.trace import status
+from opencensus.trace import time_event
+from opencensus.trace import trace_options
+from opencensus.trace import tracer
+
+# 60s is the default time for open census to call export.
+CENSUS_UPLOAD_INTERVAL_SECS = int(
+    os.environ.get("GRPC_PYTHON_CENSUS_EXPORT_UPLOAD_INTERVAL_SECS", 20)
+)
+
+
+class StackDriverAsyncTransport(async_.AsyncTransport):
+    """Wrapper class used to pass wait_period.
+
+    This is required because current StackDriver Tracing Exporter doesn't allow
+    us pass wait_period to AsyncTransport directly.
+
+    Args:
+      exporter: An opencensus.trace.base_exporter.Exporter object.
+    """
+
+    def __init__(self, exporter):
+        super().__init__(exporter, wait_period=CENSUS_UPLOAD_INTERVAL_SECS)
+
+
+class OpenCensusExporter(_observability.Exporter):
+    config: _observability_config.GcpObservabilityConfig
+    default_labels: Optional[Mapping[str, str]]
+    project_id: str
+    tracer: Optional[tracer.Tracer]
+    stats_recorder: Optional[StatsRecorder]
+    view_manager: Optional[ViewManager]
+
+    def __init__(self, config: _observability_config.GcpObservabilityConfig):
+        self.config = config.get()
+        self.default_labels = self.config.labels
+        self.project_id = self.config.project_id
+        self.tracer = None
+        self.stats_recorder = None
+        self.view_manager = None
+        self._setup_open_census_stackdriver_exporter()
+
+    def _setup_open_census_stackdriver_exporter(self) -> None:
+        if self.config.stats_enabled:
+            stats = stats_module.stats
+            self.stats_recorder = stats.stats_recorder
+            self.view_manager = stats.view_manager
+            # If testing locally please add resource="global" to Options, otherwise
+            # StackDriver might override project_id based on detected resource.
+            options = stats_exporter.Options(project_id=self.project_id)
+            metrics_exporter = stats_exporter.new_stats_exporter(
+                options, interval=CENSUS_UPLOAD_INTERVAL_SECS
+            )
+            self.view_manager.register_exporter(metrics_exporter)
+            self._register_open_census_views()
+
+        if self.config.tracing_enabled:
+            current_tracer = execution_context.get_opencensus_tracer()
+            trace_id = current_tracer.span_context.trace_id
+            span_id = current_tracer.span_context.span_id
+            if not span_id:
+                span_id = span_context_module.generate_span_id()
+            span_context = span_context_module.SpanContext(
+                trace_id=trace_id, span_id=span_id
+            )
+            # Create and Saves Tracer and Sampler to ContextVar
+            sampler = samplers.ProbabilitySampler(
+                rate=self.config.sampling_rate
+            )
+            self.trace_exporter = trace_exporter.StackdriverExporter(
+                project_id=self.project_id,
+                transport=StackDriverAsyncTransport,
+            )
+            self.tracer = tracer.Tracer(
+                sampler=sampler,
+                span_context=span_context,
+                exporter=self.trace_exporter,
+            )
+
+    def export_stats_data(
+        self, stats_data: List[_observability.StatsData]
+    ) -> None:
+        if not self.config.stats_enabled:
+            return
+        for data in stats_data:
+            measure = _views.METRICS_NAME_TO_MEASURE.get(data.name, None)
+            if not measure:
+                continue
+            # Create a measurement map for each metric, otherwise metrics will
+            # be overridden instead of accumulate.
+            measurement_map = self.stats_recorder.new_measurement_map()
+            # Add data label to default labels.
+            labels = data.labels
+            labels.update(self.default_labels)
+            tag_map = TagMap()
+            for key, value in labels.items():
+                tag_map.insert(TagKey(key), TagValue(value))
+
+            if data.measure_double:
+                measurement_map.measure_float_put(measure, data.value_float)
+            else:
+                measurement_map.measure_int_put(measure, data.value_int)
+            measurement_map.record(tag_map)
+
+    def export_tracing_data(
+        self, tracing_data: List[_observability.TracingData]
+    ) -> None:
+        if not self.config.tracing_enabled:
+            return
+        for span_data in tracing_data:
+            # Only traced data will be exported, thus TraceOptions=1.
+            span_context = span_context_module.SpanContext(
+                trace_id=span_data.trace_id,
+                span_id=span_data.span_id,
+                trace_options=trace_options.TraceOptions(1),
+            )
+            span_datas = _get_span_data(
+                span_data, span_context, self.default_labels
+            )
+            self.trace_exporter.export(span_datas)
+
+    def _register_open_census_views(self) -> None:
+        # Client
+        self.view_manager.register_view(
+            _views.client_started_rpcs(self.default_labels)
+        )
+        self.view_manager.register_view(
+            _views.client_completed_rpcs(self.default_labels)
+        )
+        self.view_manager.register_view(
+            _views.client_roundtrip_latency(self.default_labels)
+        )
+        self.view_manager.register_view(
+            _views.client_api_latency(self.default_labels)
+        )
+        self.view_manager.register_view(
+            _views.client_sent_compressed_message_bytes_per_rpc(
+                self.default_labels
+            )
+        )
+        self.view_manager.register_view(
+            _views.client_received_compressed_message_bytes_per_rpc(
+                self.default_labels
+            )
+        )
+
+        # Server
+        self.view_manager.register_view(
+            _views.server_started_rpcs(self.default_labels)
+        )
+        self.view_manager.register_view(
+            _views.server_completed_rpcs(self.default_labels)
+        )
+        self.view_manager.register_view(
+            _views.server_sent_compressed_message_bytes_per_rpc(
+                self.default_labels
+            )
+        )
+        self.view_manager.register_view(
+            _views.server_received_compressed_message_bytes_per_rpc(
+                self.default_labels
+            )
+        )
+        self.view_manager.register_view(
+            _views.server_server_latency(self.default_labels)
+        )
+
+
+def _get_span_annotations(
+    span_annotations: List[Tuple[str, str]]
+) -> List[time_event.Annotation]:
+    annotations = []
+
+    for time_stamp, description in span_annotations:
+        time = datetime.fromisoformat(time_stamp)
+        annotations.append(time_event.Annotation(time, description))
+
+    return annotations
+
+
+# pylint: disable=too-many-return-statements
+# pylint: disable=too-many-branches
+def _status_to_span_status(span_status: str) -> Optional[status.Status]:
+    if status == "OK":
+        return status.Status(code_pb2.OK, message=span_status)
+    elif status == "CANCELLED":
+        return status.Status(code_pb2.CANCELLED, message=span_status)
+    elif status == "UNKNOWN":
+        return status.Status(code_pb2.UNKNOWN, message=span_status)
+    elif status == "INVALID_ARGUMENT":
+        return status.Status(code_pb2.INVALID_ARGUMENT, message=span_status)
+    elif status == "DEADLINE_EXCEEDED":
+        return status.Status(code_pb2.DEADLINE_EXCEEDED, message=span_status)
+    elif status == "NOT_FOUND":
+        return status.Status(code_pb2.NOT_FOUND, message=span_status)
+    elif status == "ALREADY_EXISTS":
+        return status.Status(code_pb2.ALREADY_EXISTS, message=span_status)
+    elif status == "PERMISSION_DENIED":
+        return status.Status(code_pb2.PERMISSION_DENIED, message=span_status)
+    elif status == "UNAUTHENTICATED":
+        return status.Status(code_pb2.UNAUTHENTICATED, message=span_status)
+    elif status == "RESOURCE_EXHAUSTED":
+        return status.Status(code_pb2.RESOURCE_EXHAUSTED, message=span_status)
+    elif status == "FAILED_PRECONDITION":
+        return status.Status(code_pb2.FAILED_PRECONDITION, message=span_status)
+    elif status == "ABORTED":
+        return status.Status(code_pb2.ABORTED, message=span_status)
+    elif status == "OUT_OF_RANGE":
+        return status.Status(code_pb2.OUT_OF_RANGE, message=span_status)
+    elif status == "UNIMPLEMENTED":
+        return status.Status(code_pb2.UNIMPLEMENTED, message=span_status)
+    elif status == "INTERNAL":
+        return status.Status(code_pb2.INTERNAL, message=span_status)
+    elif status == "UNAVAILABLE":
+        return status.Status(code_pb2.UNAVAILABLE, message=span_status)
+    elif status == "DATA_LOSS":
+        return status.Status(code_pb2.DATA_LOSS, message=span_status)
+    else:
+        return None
+
+
+def _get_span_data(
+    span_data: _observability.TracingData,
+    span_context: span_context_module.SpanContext,
+    labels: Mapping[str, str],
+) -> List[span_data_module.SpanData]:
+    """Extracts a list of SpanData tuples from a span.
+
+    Args:
+    span_data: _observability.TracingData to convert.
+    span_context: The context related to the span_data.
+    labels: Labels to be added to SpanData.
+
+    Returns:
+    A list of opencensus.trace.span_data.SpanData.
+    """
+    span_attributes = span_data.span_labels
+    span_attributes.update(labels)
+    span_status = _status_to_span_status(span_data.status)
+    span_annotations = _get_span_annotations(span_data.span_annotations)
+    span_datas = [
+        span_data_module.SpanData(
+            name=span_data.name,
+            context=span_context,
+            span_id=span_data.span_id,
+            parent_span_id=span_data.parent_span_id
+            if span_data.parent_span_id
+            else None,
+            attributes=span_attributes,
+            start_time=span_data.start_time,
+            end_time=span_data.end_time,
+            child_span_count=span_data.child_span_count,
+            stack_trace=None,
+            annotations=span_annotations,
+            message_events=None,
+            links=None,
+            status=span_status,
+            same_process_as_parent_span=True
+            if span_data.parent_span_id
+            else None,
+            span_kind=span.SpanKind.UNSPECIFIED,
+        )
+    ]
+
+    return span_datas