port-ocean 0.21.5__py3-none-any.whl → 0.22.1__py3-none-any.whl

port_ocean/helpers/metric/metric.py

@@ -0,0 +1,238 @@
+from typing import Any, TYPE_CHECKING, Optional, Dict, List, Tuple
+from fastapi import APIRouter
+from port_ocean.exceptions.context import ResourceContextNotFoundError
+import prometheus_client
+from httpx import AsyncClient
+
+from loguru import logger
+from port_ocean.context import resource
+from prometheus_client import Gauge
+import prometheus_client.openmetrics
+import prometheus_client.openmetrics.exposition
+import prometheus_client.parser
+
+if TYPE_CHECKING:
+    from port_ocean.config.settings import MetricsSettings, IntegrationSettings
+
+
+class MetricPhase:
+    EXTRACT = "extract"
+    TRANSFORM = "transform"
+    LOAD = "load"
+    RESYNC = "resync"
+    DELETE = "delete"
+
+
+class MetricType:
+    # Define metric names as constants
+    DURATION_NAME = "duration_seconds"
+    OBJECT_COUNT_NAME = "object_count"
+    ERROR_COUNT_NAME = "error_count"
+    SUCCESS_NAME = "success"
+    RATE_LIMIT_WAIT_NAME = "rate_limit_wait_seconds"
+    DELETION_COUNT_NAME = "deletion_count"
+
+
+# Registry for core and custom metrics
+_metrics_registry: Dict[str, Tuple[str, str, List[str]]] = {
+    MetricType.DURATION_NAME: (
+        MetricType.DURATION_NAME,
+        "duration description",
+        ["kind", "phase"],
+    ),
+    MetricType.OBJECT_COUNT_NAME: (
+        MetricType.OBJECT_COUNT_NAME,
+        "object_count description",
+        ["kind", "phase"],
+    ),
+    MetricType.ERROR_COUNT_NAME: (
+        MetricType.ERROR_COUNT_NAME,
+        "error_count description",
+        ["kind", "phase"],
+    ),
+    MetricType.SUCCESS_NAME: (
+        MetricType.SUCCESS_NAME,
+        "success description",
+        ["kind", "phase"],
+    ),
+    MetricType.RATE_LIMIT_WAIT_NAME: (
+        MetricType.RATE_LIMIT_WAIT_NAME,
+        "rate_limit_wait description",
+        ["kind", "phase", "endpoint"],
+    ),
+    MetricType.DELETION_COUNT_NAME: (
+        MetricType.DELETION_COUNT_NAME,
+        "deletion_count description",
+        ["kind", "phase"],
+    ),
+}
+
+
+def register_metric(name: str, description: str, labels: List[str]) -> None:
+    """Register a custom metric that will be available for use.
+
+    Args:
+        name (str): The metric name to register
+        description (str): Description of what the metric measures
+        labels (list[str]): Labels to apply to the metric
+    """
+    _metrics_registry[name] = (name, description, labels)
+
+
+class EmptyMetric:
+    def set(self, *args: Any) -> None:
+        return None
+
+    def labels(self, *args: Any) -> None:
+        return None
+
+
+class Metrics:
+    def __init__(
+        self,
+        metrics_settings: "MetricsSettings",
+        integration_configuration: "IntegrationSettings",
+    ) -> None:
+        self.metrics_settings = metrics_settings
+        self.integration_configuration = integration_configuration
+        self.registry = prometheus_client.CollectorRegistry()
+        self.metrics: dict[str, Gauge] = {}
+        self.load_metrics()
+        self._integration_version: Optional[str] = None
+        self._ocean_version: Optional[str] = None
+
+    @property
+    def integration_version(self) -> str:
+        if self._integration_version is None:
+            from port_ocean.version import __integration_version__
+
+            self._integration_version = __integration_version__
+        return self._integration_version
+
+    @property
+    def ocean_version(self) -> str:
+        if self._ocean_version is None:
+            from port_ocean.version import __version__
+
+            self._ocean_version = __version__
+        return self._ocean_version
+
+    @property
+    def enabled(self) -> bool:
+        return self.metrics_settings.enabled
+
+    def load_metrics(self) -> None:
+        if not self.enabled:
+            return None
+
+        # Load all registered metrics
+        for name, (_, description, labels) in _metrics_registry.items():
+            self.metrics[name] = Gauge(
+                name, description, labels, registry=self.registry
+            )
+
+    def get_metric(self, name: str, labels: list[str]) -> Gauge | EmptyMetric:
+        if not self.enabled:
+            return EmptyMetric()
+        metrics = self.metrics.get(name)
+        if not metrics:
+            return EmptyMetric()
+        return metrics.labels(*labels)
+
+    def set_metric(self, name: str, labels: list[str], value: float) -> None:
+        """Set a metric value in a single method call.
+
+        Args:
+            name (str): The metric name to set.
+            labels (list[str]): The labels to apply to the metric.
+            value (float): The value to set.
+        """
+        if not self.enabled:
+            return None
+
+        self.get_metric(name, labels).set(value)
+
+    def create_mertic_router(self) -> APIRouter:
+        if not self.enabled:
+            return APIRouter()
+        router = APIRouter()
+
+        @router.get("/")
+        async def prom_metrics() -> str:
+            return self.generate_latest()
+
+        return router
+
+    def current_resource_kind(self) -> str:
+        try:
+            return f"{resource.resource.kind}-{resource.resource.index}"
+        except ResourceContextNotFoundError:
+            return "__runtime__"
+
+    def generate_latest(self) -> str:
+        return prometheus_client.openmetrics.exposition.generate_latest(
+            self.registry
+        ).decode()
+
+    async def flush(
+        self, metric_name: Optional[str] = None, kind: Optional[str] = None
+    ) -> None:
+        if not self.enabled:
+            return None
+
+        if not self.metrics_settings.webhook_url:
+            return None
+
+        try:
+            latest_raw = self.generate_latest()
+            metric_families = prometheus_client.parser.text_string_to_metric_families(
+                latest_raw
+            )
+            metrics_dict: dict[str, Any] = {}
+            for family in metric_families:
+                for sample in family.samples:
+                    # Skip if a specific metric name was requested and this isn't it
+                    if metric_name and sample.name != metric_name:
+                        continue
+
+                    current_level = metrics_dict
+                    if sample.labels:
+                        # Skip if a specific kind was requested and this isn't it
+                        if kind and sample.labels.get("kind") != kind:
+                            continue
+
+                        # Create nested dictionary structure based on labels
+                        for key, value in sample.labels.items():
+                            if key not in current_level:
+                                current_level[key] = {}
+                            current_level = current_level[key]
+                            if value not in current_level:
+                                current_level[value] = {}
+                            current_level = current_level[value]
+
+                    current_level[sample.name] = sample.value
+
+            # If no metrics were filtered, exit early
+            if not metrics_dict.get("kind", {}):
+                return None
+
+            for kind_key, metrics in metrics_dict.get("kind", {}).items():
+                # Skip if we're filtering by kind and this isn't the requested kind
+                if kind and kind_key != kind:
+                    continue
+
+                event = {
+                    "integration_type": self.integration_configuration.type,
+                    "integration_identifier": self.integration_configuration.identifier,
+                    "integration_version": self.integration_version,
+                    "ocean_version": self.ocean_version,
+                    "kind_identifier": kind_key,
+                    "kind": "-".join(kind_key.split("-")[:-1]),
+                    "metrics": metrics,
+                }
+                logger.info(f"Sending metrics to webhook {kind_key}: {event}")
+                await AsyncClient().post(
+                    url=self.metrics_settings.webhook_url, json=event
+                )
+        except Exception as e:
+            logger.error(f"Error sending metrics to webhook: {e}")

port_ocean/helpers/metric/utils.py

@@ -0,0 +1,30 @@
+from functools import wraps
+import time
+from typing import Any, Callable
+
+from port_ocean.context.ocean import ocean
+from port_ocean.helpers.metric.metric import MetricType
+
+
+def TimeMetric(phase: str) -> Any:
+    def decorator(func: Callable[..., Any]) -> Any:
+
+        @wraps(func)
+        async def wrapper(*args: Any, **kwargs: dict[Any, Any]) -> Any:
+            if not ocean.metrics.enabled:
+                return await func(*args, **kwargs)
+            start = time.monotonic()
+            res = await func(*args, **kwargs)
+            end = time.monotonic()
+            duration = end - start
+            ocean.metrics.set_metric(
+                name=MetricType.DURATION_NAME,
+                labels=[ocean.metrics.current_resource_kind(), phase],
+                value=duration,
+            )
+
+            return res
+
+        return wrapper
+
+    return decorator

port_ocean/helpers/retry.py

@@ -5,7 +5,6 @@ from datetime import datetime
 from functools import partial
 from http import HTTPStatus
 from typing import Any, Callable, Coroutine, Iterable, Mapping, Union
-
 import httpx
 from dateutil.parser import isoparse
 
@@ -172,6 +171,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
                 response = await self._retry_operation_async(request, send_method)
             else:
                 response = await transport.handle_async_request(request)
+
             return response
         except Exception as e:
             # Retyable methods are logged via _log_error
@@ -340,6 +340,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
         attempts_made = 0
         response: httpx.Response | None = None
         error: Exception | None = None
+
         while True:
             if attempts_made > 0:
                 sleep_time = self._calculate_sleep(attempts_made, {})

port_ocean/ocean.py

@@ -4,7 +4,10 @@ from contextlib import asynccontextmanager
 import threading
 from typing import Any, AsyncIterator, Callable, Dict, Type
 
-from fastapi import APIRouter, FastAPI
+import port_ocean.helpers.metric.metric
+
+from fastapi import FastAPI, APIRouter
+
 from loguru import logger
 from pydantic import BaseModel
 from starlette.types import Receive, Scope, Send
@@ -49,12 +52,15 @@ class Ocean:
             _integration_config_model=config_factory,
             **(config_override or {}),
         )
-
         # add the integration sensitive configuration to the sensitive patterns to mask out
         sensitive_log_filter.hide_sensitive_strings(
             *self.config.get_sensitive_fields_data()
         )
         self.integration_router = integration_router or APIRouter()
+        self.metrics = port_ocean.helpers.metric.metric.Metrics(
+            metrics_settings=self.config.metrics,
+            integration_configuration=self.config.integration,
+        )
 
         self.webhook_manager = LiveEventsProcessorManager(
             self.integration_router,
@@ -91,8 +97,12 @@ class Ocean:
             await self.resync_state_updater.update_before_resync()
             logger.info("Starting a new scheduled resync")
             try:
-                await self.integration.sync_raw_all()
-                await self.resync_state_updater.update_after_resync()
+                successed = await self.integration.sync_raw_all()
+                await self.resync_state_updater.update_after_resync(
+                    IntegrationStateStatus.Completed
+                    if successed
+                    else IntegrationStateStatus.Failed
+                )
             except asyncio.CancelledError:
                 logger.warning(
                     "resync was cancelled by the scheduled resync, skipping state update"
@@ -148,6 +158,9 @@ class Ocean:
 
     def initialize_app(self) -> None:
         self.fast_api_app.include_router(self.integration_router, prefix="/integration")
+        self.fast_api_app.include_router(
+            self.metrics.create_mertic_router(), prefix="/metrics"
+        )
 
         @asynccontextmanager
         async def lifecycle(_: FastAPI) -> AsyncIterator[None]:

port_ocean/tests/clients/port/mixins/test_entities.py

@@ -1,5 +1,5 @@
 from typing import Any
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
 
@@ -37,17 +37,20 @@ async def entity_client(monkeypatch: Any) -> EntityClientMixin:
 async def test_batch_upsert_entities_read_timeout_should_raise_false(
     entity_client: EntityClientMixin,
 ) -> None:
-    result_entities = await entity_client.batch_upsert_entities(
-        entities=all_entities, request_options=MagicMock(), should_raise=False
-    )
+    with patch("port_ocean.context.event.event", MagicMock()):
+        result_entities = await entity_client.batch_upsert_entities(
+            entities=all_entities, request_options=MagicMock(), should_raise=False
+        )
+        entities_only = [entity for _, entity in result_entities]
 
-    assert result_entities == expected_result_entities
+        assert entities_only == expected_result_entities
 
 
 async def test_batch_upsert_entities_read_timeout_should_raise_true(
     entity_client: EntityClientMixin,
 ) -> None:
-    with pytest.raises(ReadTimeout):
-        await entity_client.batch_upsert_entities(
-            entities=all_entities, request_options=MagicMock(), should_raise=True
-        )
+    with patch("port_ocean.context.event.event", MagicMock()):
+        with pytest.raises(ReadTimeout):
+            await entity_client.batch_upsert_entities(
+                entities=all_entities, request_options=MagicMock(), should_raise=True
+            )

port_ocean/tests/core/conftest.py

@@ -0,0 +1,187 @@
+from contextlib import asynccontextmanager
+from typing import Any, AsyncGenerator
+from unittest.mock import MagicMock, AsyncMock, patch
+
+import pytest
+from httpx import Response
+
+from port_ocean.clients.port.client import PortClient
+from port_ocean.context.event import EventContext
+from port_ocean.context.ocean import PortOceanContext, ocean
+from port_ocean.core.handlers.entities_state_applier.port.applier import (
+    HttpEntitiesStateApplier,
+)
+from port_ocean.core.handlers.entity_processor.jq_entity_processor import (
+    JQEntityProcessor,
+)
+from port_ocean.core.handlers.port_app_config.models import (
+    EntityMapping,
+    MappingsConfig,
+    PortAppConfig,
+    PortResourceConfig,
+    ResourceConfig,
+    Selector,
+)
+from port_ocean.core.models import Entity
+from port_ocean.ocean import Ocean
+
+
+@pytest.fixture
+def mock_http_client() -> MagicMock:
+    mock_http_client = MagicMock()
+    mock_upserted_entities = []
+
+    async def post(url: str, *args: Any, **kwargs: Any) -> Response:
+        entity = kwargs.get("json", {})
+        if entity.get("properties", {}).get("mock_is_to_fail", {}):
+            return Response(
+                404, headers=MagicMock(), json={"ok": False, "error": "not_found"}
+            )
+
+        mock_upserted_entities.append(
+            f"{entity.get('identifier')}-{entity.get('blueprint')}"
+        )
+        return Response(
+            200,
+            json={
+                "entity": {
+                    "identifier": entity.get("identifier"),
+                    "blueprint": entity.get("blueprint"),
+                }
+            },
+        )
+
+    mock_http_client.post = AsyncMock(side_effect=post)
+    return mock_http_client
+
+
+@pytest.fixture
+def mock_port_client(mock_http_client: MagicMock) -> PortClient:
+    mock_port_client = PortClient(
+        MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
+    )
+    mock_port_client.auth = AsyncMock()
+    mock_port_client.auth.headers = AsyncMock(
+        return_value={
+            "Authorization": "test",
+            "User-Agent": "test",
+        }
+    )
+
+    mock_port_client.search_entities = AsyncMock(return_value=[])  # type: ignore
+    mock_port_client.client = mock_http_client
+    return mock_port_client
+
+
+@pytest.fixture
+def mock_ocean(mock_port_client: PortClient) -> Ocean:
+    with patch("port_ocean.ocean.Ocean.__init__", return_value=None):
+        ocean_mock = Ocean(
+            MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock()
+        )
+        ocean_mock.config = MagicMock()
+        ocean_mock.config.port = MagicMock()
+        ocean_mock.config.port.port_app_config_cache_ttl = 60
+        ocean_mock.port_client = mock_port_client
+        ocean_mock.metrics = MagicMock()
+        ocean_mock.metrics.flush = AsyncMock()
+
+        return ocean_mock
+
+
+@pytest.fixture
+def mock_context(mock_ocean: Ocean) -> PortOceanContext:
+    context = PortOceanContext(mock_ocean)
+    ocean._app = context.app
+    return context
+
+
+@pytest.fixture
+def mock_port_app_config() -> PortAppConfig:
+    return PortAppConfig(
+        enable_merge_entity=True,
+        delete_dependent_entities=True,
+        create_missing_related_entities=False,
+        resources=[
+            ResourceConfig(
+                kind="project",
+                selector=Selector(query="true"),
+                port=PortResourceConfig(
+                    entity=MappingsConfig(
+                        mappings=EntityMapping(
+                            identifier=".id | tostring",
+                            title=".name",
+                            blueprint='"service"',
+                            properties={"url": ".web_url"},
+                            relations={},
+                        )
+                    )
+                ),
+            )
+        ],
+    )
+
+
+@pytest.fixture
+def mock_port_app_config_handler(mock_port_app_config: PortAppConfig) -> MagicMock:
+    handler = MagicMock()
+
+    async def get_config(use_cache: bool = True) -> Any:
+        return mock_port_app_config
+
+    handler.get_port_app_config = get_config
+    return handler
+
+
+@pytest.fixture
+def mock_entity_processor(mock_context: PortOceanContext) -> JQEntityProcessor:
+    return JQEntityProcessor(mock_context)
+
+
+@pytest.fixture
+def mock_resource_config() -> ResourceConfig:
+    resource = ResourceConfig(
+        kind="service",
+        selector=Selector(query="true"),
+        port=PortResourceConfig(
+            entity=MappingsConfig(
+                mappings=EntityMapping(
+                    identifier=".id",
+                    title=".name",
+                    blueprint='"service"',
+                    properties={"url": ".web_url"},
+                    relations={},
+                )
+            )
+        ),
+    )
+    return resource
+
+
+@pytest.fixture
+def mock_entities_state_applier(
+    mock_context: PortOceanContext,
+) -> HttpEntitiesStateApplier:
+    return HttpEntitiesStateApplier(mock_context)
+
+
+@asynccontextmanager
+async def no_op_event_context(
+    existing_event: EventContext,
+) -> AsyncGenerator[EventContext, None]:
+    yield existing_event
+
+
+def create_entity(
+    id: str,
+    blueprint: str,
+    relation: dict[str, str] | None = None,
+    is_to_fail: bool = False,
+    team: str | list[str] | dict[str, Any] | None = None,
+) -> Entity:
+    if relation is None:
+        relation = {}
+    entity = Entity(identifier=id, blueprint=blueprint, team=team)
+    entity.relations = relation
+    entity.properties = {"mock_is_to_fail": is_to_fail}
+    return entity