port-ocean 0.23.3__tar.gz → 0.23.4__tar.gz

{port_ocean-0.23.3 → port_ocean-0.23.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.23.3
+Version: 0.23.4
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.23.3 → port_ocean-0.23.4}/port_ocean/clients/port/mixins/integrations.py

@@ -214,8 +214,9 @@ class IntegrationClientMixin:
         logger.debug("starting POST metrics request", metrics=metrics)
         metrics_attributes = await self.get_metrics_attributes()
         headers = await self.auth.headers()
+        url = metrics_attributes["ingestUrl"] + "/syncMetrics"
         response = await self.client.post(
-            metrics_attributes["ingestUrl"],
+            url,
             headers=headers,
             json={
                 "syncKindsMetrics": metrics,
@@ -229,7 +230,7 @@ class IntegrationClientMixin:
         metrics_attributes = await self.get_metrics_attributes()
         url = (
             metrics_attributes["ingestUrl"]
-            + f"/resync/{kind_metrics['eventId']}/kind/{kind_metrics['kindIdentifier']}"
+            + f"/syncMetrics/resync/{kind_metrics['eventId']}/kind/{kind_metrics['kindIdentifier']}"
         )
         headers = await self.auth.headers()
         response = await self.client.put(

port_ocean-0.23.4/port_ocean/context/metric_resource.py

@@ -0,0 +1,63 @@
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import AsyncIterator, TYPE_CHECKING
+
+from loguru import logger
+from werkzeug.local import LocalStack, LocalProxy
+
+from port_ocean.exceptions.context import (
+    ResourceContextNotFoundError,
+)
+
+if TYPE_CHECKING:
+    pass
+
+
+@dataclass
+class MetricResourceContext:
+    """
+    The metric resource context is a context manager that allows you to access the current metric resource if there is one.
+    This is useful for getting the metric resource kind
+    """
+
+    metric_resource_kind: str
+    index: int
+
+    @property
+    def kind(self) -> str:
+        return self.metric_resource_kind
+
+
+_resource_context_stack: LocalStack[MetricResourceContext] = LocalStack()
+
+
+def _get_metric_resource_context() -> MetricResourceContext:
+    """
+    Get the context from the current thread.
+    """
+    top_resource_context = _resource_context_stack.top
+    if top_resource_context is None:
+        raise ResourceContextNotFoundError(
+            "You must be inside an metric resource context in order to use it"
+        )
+
+    return top_resource_context
+
+
+metric_resource: MetricResourceContext = LocalProxy(lambda: _get_metric_resource_context())  # type: ignore
+
+
+@asynccontextmanager
+async def metric_resource_context(
+    metric_resource_kind: str, index: int = 0
+) -> AsyncIterator[MetricResourceContext]:
+    _resource_context_stack.push(
+        MetricResourceContext(metric_resource_kind=metric_resource_kind, index=index)
+    )
+
+    with logger.contextualize(
+        metric_resource_kind=metric_resource.metric_resource_kind
+    ):
+        yield metric_resource
+
+    _resource_context_stack.pop()

{port_ocean-0.23.3 → port_ocean-0.23.4}/port_ocean/core/handlers/resync_state_updater/updater.py

@@ -94,6 +94,6 @@ class ResyncStateUpdater:
         await ocean.metrics.send_metrics_to_webhook(
             kind=ocean.metrics.current_resource_kind()
         )
-        # await ocean.metrics.report_sync_metrics(
-        #     kinds=[ocean.metrics.current_resource_kind()]
-        # ) # TODO: uncomment this when end points are ready
+        await ocean.metrics.report_sync_metrics(
+            kinds=[ocean.metrics.current_resource_kind()]
+        )

{port_ocean-0.23.3 → port_ocean-0.23.4}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -9,6 +9,7 @@ import httpx
 from loguru import logger
 from port_ocean.clients.port.types import UserAgentType
 from port_ocean.context.event import TriggerType, event_context, EventType, event
+from port_ocean.context.metric_resource import metric_resource_context
 from port_ocean.context.ocean import ocean
 from port_ocean.context.resource import resource_context
 from port_ocean.context import resource
@@ -33,8 +34,8 @@ from port_ocean.core.ocean_types import (
 )
 from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
-from port_ocean.helpers.metric.metric import SyncState, MetricType, MetricPhase
-from port_ocean.helpers.metric.utils import TimeMetric
+from port_ocean.helpers.metric.metric import MetricResourceKind, SyncState, MetricType, MetricPhase
+from port_ocean.helpers.metric.utils import TimeMetric, TimeMetricWithResourceKind
 from port_ocean.utils.ipc import FileIPC
 
 SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
@@ -249,9 +250,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                             labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.SKIPPED],
                             value=len(objects_diff[0].entity_selector_diff.passed) - len(changed_entities)
                         )
-                    await self.entities_state_applier.upsert(
+                    upserted_entities = await self.entities_state_applier.upsert(
                         changed_entities, user_agent_type
                     )
+
+                    ocean.metrics.set_metric(
+                        name=MetricType.OBJECT_COUNT_NAME,
+                        labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.LOADED],
+                        value=len(upserted_entities)
+                    )
+
                 else:
                     logger.info("Entities in batch didn't changed since last sync, skipping", total_entities=len(objects_diff[0].entity_selector_diff.passed))
                     ocean.metrics.inc_metric(
@@ -265,6 +273,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 modified_objects = await self.entities_state_applier.upsert(
                     objects_diff[0].entity_selector_diff.passed, user_agent_type
                     )
+                ocean.metrics.set_metric(
+                        name=MetricType.OBJECT_COUNT_NAME,
+                        labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.LOADED],
+                        value=len(upserted_entities)
+                    )
         else:
            modified_objects = await self.entities_state_applier.upsert(
                     objects_diff[0].entity_selector_diff.passed, user_agent_type
@@ -633,16 +646,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         async with resource_context(resource,index):
             resource_kind_id = f"{resource.kind}-{index}"
             ocean.metrics.sync_state = SyncState.SYNCING
+
             task = asyncio.create_task(
                 self._register_in_batches(resource, user_agent_type)
             )
             event.on_abort(lambda: task.cancel())
             kind_results: tuple[list[Entity], list[Exception]] = await task
-            ocean.metrics.set_metric(
-                name=MetricType.OBJECT_COUNT_NAME,
-                labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.LOADED],
-                value=len(kind_results[0])
-            )
 
             if ocean.metrics.sync_state != SyncState.FAILED:
                 ocean.metrics.sync_state = SyncState.COMPLETED
@@ -650,10 +659,88 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             await ocean.metrics.send_metrics_to_webhook(
                 kind=resource_kind_id
             )
-            # await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
+            await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id, blueprint=resource.port.entity.mappings.blueprint)
 
             return kind_results
 
+    @TimeMetricWithResourceKind(MetricPhase.RESYNC)
+    async def resync_reconciliation(
+        self,
+        creation_results: list[tuple[list[Entity], list[Exception]]],
+        did_fetched_current_state: bool,
+        user_agent_type: UserAgentType,
+        app_config: Any,
+        silent: bool = True,
+    ) -> None:
+        """Handle the reconciliation phase of the resync process.
+
+        This method handles:
+        1. Sorting and upserting failed entities
+        2. Checking if current state was fetched
+        3. Calculating resync diff
+        4. Handling errors
+        5. Deleting entities that are no longer needed
+        6. Executing resync complete hooks
+
+        Args:
+            creation_results (list[tuple[list[Entity], list[Exception]]]): Results from entity creation
+            did_fetched_current_state (bool): Whether the current state was successfully fetched
+            user_agent_type (UserAgentType): The type of user agent
+            app_config (Any): The application configuration
+            silent (bool): Whether to raise exceptions or handle them silently
+
+        """
+        await self.sort_and_upsert_failed_entities(user_agent_type)
+
+        if not did_fetched_current_state:
+            logger.warning(
+                "Due to an error before the resync, the previous state of entities at Port is unknown."
+                " Skipping delete phase due to unknown initial state."
+            )
+            return False
+
+        logger.info("Starting resync diff calculation")
+        generated_entities, errors = zip_and_sum(creation_results) or [
+            [],
+            [],
+        ]
+
+        if errors:
+            message = f"Resync failed with {len(errors)} errors, skipping delete phase due to incomplete state"
+            error_group = ExceptionGroup(
+                message,
+                errors,
+            )
+            if not silent:
+                raise error_group
+
+            logger.error(message, exc_info=error_group)
+            return False
+
+        logger.info(
+            f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
+        )
+        entities_at_port = await ocean.port_client.search_entities(
+            user_agent_type
+        )
+
+        await self.entities_state_applier.delete_diff(
+            {"before": entities_at_port, "after": generated_entities},
+            user_agent_type, app_config.get_entity_deletion_threshold()
+        )
+
+        logger.info("Resync finished successfully")
+
+        # Execute resync_complete hooks
+        if "resync_complete" in self.event_strategy:
+            logger.info("Executing resync_complete hooks")
+
+            for resync_complete_fn in self.event_strategy["resync_complete"]:
+                await resync_complete_fn()
+
+            logger.info("Finished executing resync_complete hooks")
+
+
     @TimeMetric(MetricPhase.RESYNC)
     async def sync_raw_all(
         self,
@@ -689,8 +776,9 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             logger.info(f"Resync will use the following mappings: {app_config.dict()}")
 
             kinds = [f"{resource.kind}-{index}" for index, resource in enumerate(app_config.resources)]
+            blueprints = [resource.port.entity.mappings.blueprint for resource in app_config.resources]
             ocean.metrics.initialize_metrics(kinds)
-            # await ocean.metrics.report_sync_metrics(kinds=kinds) # TODO: uncomment this when end points are ready
+            await ocean.metrics.report_sync_metrics(kinds=kinds, blueprints=blueprints)
 
             # Clear cache
             await ocean.app.cache_provider.clear()
@@ -716,65 +804,20 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             multiprocessing.set_start_method('fork', True)
             try:
                 for index,resource in enumerate(app_config.resources):
-
                     logger.info(f"Starting processing resource {resource.kind} with index {index}")
-
                     creation_results.append(await self.process_resource(resource,index,user_agent_type))
-
-                await self.sort_and_upsert_failed_entities(user_agent_type)
-
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
                 raise
             else:
-                if not did_fetched_current_state:
-                    logger.warning(
-                        "Due to an error before the resync, the previous state of entities at Port is unknown."
-                        " Skipping delete phase due to unknown initial state."
-                    )
-                    return
-
-                logger.info("Starting resync diff calculation")
-                generated_entities, errors = zip_and_sum(creation_results) or [
-                    [],
-                    [],
-                ]
-
-                if errors:
-                    message = f"Resync failed with {len(errors)} errors, skipping delete phase due to incomplete state"
-                    error_group = ExceptionGroup(
-                        message,
-                        errors,
-                    )
-                    if not silent:
-                        raise error_group
-
-                    logger.error(message, exc_info=error_group)
-                    return False
-                else:
-                    logger.info(
-                        f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
-                    )
-                    entities_at_port = await ocean.port_client.search_entities(
-                        user_agent_type
-                    )
-                    await self.entities_state_applier.delete_diff(
-                        {"before": entities_at_port, "after": generated_entities},
-                        user_agent_type, app_config.get_entity_deletion_threshold()
-                    )
-
-                    logger.info("Resync finished successfully")
-
-                    # Execute resync_complete hooks
-                    if "resync_complete" in self.event_strategy:
-                        logger.info("Executing resync_complete hooks")
-
-                        for resync_complete_fn in self.event_strategy["resync_complete"]:
-                            await resync_complete_fn()
-
-                        logger.info("Finished executing resync_complete hooks")
-
-                    return True
+                await self.resync_reconciliation(
+                    creation_results,
+                    did_fetched_current_state,
+                    user_agent_type,
+                    app_config,
+                    silent
+                )
+                await ocean.metrics.report_sync_metrics(kinds=[MetricResourceKind.RECONCILIATION])
             finally:
                 await ocean.app.cache_provider.clear()
                 if ocean.app.process_execution_mode == ProcessExecutionMode.multi_process:

{port_ocean-0.23.3 → port_ocean-0.23.4}/port_ocean/helpers/metric/metric.py

@@ -6,7 +6,7 @@ import prometheus_client
 from httpx import AsyncClient
 from fastapi.responses import PlainTextResponse
 from loguru import logger
-from port_ocean.context import resource
+from port_ocean.context import metric_resource, resource
 from prometheus_client import Gauge
 import prometheus_client.openmetrics
 import prometheus_client.openmetrics.exposition
@@ -57,6 +57,11 @@ class SyncState:
     FAILED = "failed"
 
 
+class MetricResourceKind:
+    RECONCILIATION = "__reconciliation__"
+    RESYNC = "__resync__"
+
+
 # Registry for core and custom metrics
 _metrics_registry: Dict[str, Tuple[str, str, List[str]]] = {
     MetricType.DURATION_NAME: (
@@ -252,8 +257,6 @@ class Metrics:
             )
 
     def create_mertic_router(self) -> APIRouter:
-        if not self.enabled:
-            return APIRouter()
         router = APIRouter()
 
         @router.get("/", response_class=PlainTextResponse)
@@ -265,6 +268,12 @@ class Metrics:
     def current_resource_kind(self) -> str:
         try:
             return f"{resource.resource.kind}-{resource.resource.index}"
+        except ResourceContextNotFoundError:
+            return self.current_metric_resource_kind()
+
+    def current_metric_resource_kind(self) -> str:
+        try:
+            return metric_resource.metric_resource.metric_resource_kind
         except ResourceContextNotFoundError:
             return "__runtime__"
 
@@ -274,15 +283,21 @@ class Metrics:
         ).decode()
 
     async def report_sync_metrics(
-        self, metric_name: Optional[str] = None, kinds: Optional[list[str]] = None
+        self,
+        metric_name: Optional[str] = None,
+        kinds: Optional[list[str]] = None,
+        blueprints: Optional[list[Optional[str]]] = None,
     ) -> None:
         if kinds is None:
             return None
 
         metrics = []
 
-        for kind in kinds:
-            metric = self.generate_metrics(metric_name, kind)
+        if blueprints is None:
+            blueprints = [None] * len(kinds)
+
+        for kind, blueprint in zip(kinds, blueprints):
+            metric = self.generate_metrics(metric_name, kind, blueprint)
             metrics.extend(metric)
 
         try:
@@ -291,9 +306,12 @@ class Metrics:
             logger.error(f"Error posting metrics: {e}", metrics=metrics)
 
     async def report_kind_sync_metrics(
-        self, metric_name: Optional[str] = None, kind: Optional[str] = None
+        self,
+        metric_name: Optional[str] = None,
+        kind: Optional[str] = None,
+        blueprint: Optional[str] = None,
     ) -> None:
-        metrics = self.generate_metrics(metric_name, kind)
+        metrics = self.generate_metrics(metric_name, kind, blueprint)
         if not metrics:
             return None
 
@@ -304,7 +322,10 @@ class Metrics:
             logger.error(f"Error putting metrics: {e}", metrics=metrics)
 
     def generate_metrics(
-        self, metric_name: Optional[str] = None, kind: Optional[str] = None
+        self,
+        metric_name: Optional[str] = None,
+        kind: Optional[str] = None,
+        blueprint: Optional[str] = None,
     ) -> list[dict[str, Any]]:
         try:
             latest_raw = self.generate_latest()
@@ -363,9 +384,10 @@ class Metrics:
                         if "-" in kind_key
                         else kind_key
                     ),
-                    "kindIndex": 0 if kind_key == "__runtime__" else int(kind_key[-1]),
+                    "kindIndex": int(kind_key[-1]) if kind_key[-1].isdigit() else 0,
                     "eventId": self.event_id,
                     "syncState": self.sync_state,
+                    "blueprint": blueprint if blueprint else "",
                     "metrics": metrics,
                 }
                 events.append(event)

port_ocean-0.23.4/port_ocean/helpers/metric/utils.py

@@ -0,0 +1,52 @@
+from functools import wraps
+import time
+from typing import Any, Callable
+
+from port_ocean.context.metric_resource import metric_resource_context
+from port_ocean.context.ocean import ocean
+from port_ocean.helpers.metric.metric import MetricResourceKind, MetricType
+
+
+def TimeMetric(phase: str) -> Any:
+    def decorator(func: Callable[..., Any]) -> Any:
+
+        @wraps(func)
+        async def wrapper(*args: Any, **kwargs: dict[Any, Any]) -> Any:
+            start = time.monotonic()
+            res = await func(*args, **kwargs)
+            end = time.monotonic()
+            duration = end - start
+            ocean.metrics.inc_metric(
+                name=MetricType.DURATION_NAME,
+                labels=[ocean.metrics.current_resource_kind(), phase],
+                value=duration,
+            )
+
+            return res
+
+        return wrapper
+
+    return decorator
+
+
+def TimeMetricWithResourceKind(phase: str) -> Any:
+    def decorator(func: Callable[..., Any]) -> Any:
+
+        @wraps(func)
+        async def wrapper(*args: Any, **kwargs: dict[Any, Any]) -> Any:
+            async with metric_resource_context(MetricResourceKind.RECONCILIATION):
+                start = time.monotonic()
+                res = await func(*args, **kwargs)
+                end = time.monotonic()
+                duration = end - start
+                ocean.metrics.inc_metric(
+                    name=MetricType.DURATION_NAME,
+                    labels=[ocean.metrics.current_resource_kind(), phase],
+                    value=duration,
+                )
+
+                return res
+
+        return wrapper
+
+    return decorator

{port_ocean-0.23.3 → port_ocean-0.23.4}/port_ocean/tests/core/handlers/mixins/test_sync_raw.py

@@ -154,7 +154,7 @@ async def test_sync_raw_mixin_self_dependency(
 
                 # Add assertions for actual metrics
                 metrics = mock_ocean.metrics.generate_metrics()
-                assert len(metrics) == 2
+                assert len(metrics) == 3
 
                 # Verify object counts
                 for metric in metrics:
@@ -187,7 +187,7 @@ async def test_sync_raw_mixin_self_dependency(
                             metric["metrics"]["phase"]["load"]["object_count_type"][
                                 "loaded"
                             ]["object_count"]
-                            == 2
+                            == 1
                         )
 
                         # Verify success
@@ -196,6 +196,14 @@ async def test_sync_raw_mixin_self_dependency(
                         # Verify sync state
                         assert metric["syncState"] == "completed"
 
+                    if metric["kind"] == "reconciliation":
+                        assert (
+                            metric["metrics"]["phase"]["load"]["object_count_type"][
+                                "failed"
+                            ]["object_count"]
+                            == 1
+                        )
+
 
 @pytest.mark.asyncio
 async def test_sync_raw_mixin_circular_dependency(
@@ -282,7 +290,7 @@ async def test_sync_raw_mixin_circular_dependency(
 
                 # Add assertions for actual metrics
                 metrics = mock_ocean.metrics.generate_metrics()
-                assert len(metrics) == 2
+                assert len(metrics) == 3
 
                 # Verify object counts
                 for metric in metrics:
@@ -315,7 +323,7 @@ async def test_sync_raw_mixin_circular_dependency(
                             metric["metrics"]["phase"]["load"]["object_count_type"][
                                 "loaded"
                             ]["object_count"]
-                            == 2
+                            == 0
                         )
 
                         # Verify success
@@ -324,6 +332,14 @@ async def test_sync_raw_mixin_circular_dependency(
                         # Verify sync state
                         assert metric["syncState"] == "completed"
 
+                    if metric["kind"] == "reconciliation":
+                        assert (
+                            metric["metrics"]["phase"]["load"]["object_count_type"][
+                                "loaded"
+                            ]["object_count"]
+                            == 2
+                        )
+
 
 @pytest.mark.asyncio
 async def test_sync_raw_mixin_dependency(
@@ -422,7 +438,7 @@ async def test_sync_raw_mixin_dependency(
 
                 # Add assertions for actual metrics
                 metrics = mock_ocean.metrics.generate_metrics()
-                assert len(metrics) == 2
+                assert len(metrics) == 3
 
                 # Verify object counts
                 for metric in metrics:
@@ -458,6 +474,14 @@ async def test_sync_raw_mixin_dependency(
                         # Verify sync state
                         assert metric["syncState"] == "completed"
 
+                    if metric["kind"] == "reconciliation":
+                        assert (
+                            metric["metrics"]["phase"]["load"]["object_count_type"][
+                                "loaded"
+                            ]["object_count"]
+                            == 5
+                        )
+
 
 @pytest.mark.asyncio
 async def test_register_raw(

{port_ocean-0.23.3 → port_ocean-0.23.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.23.3"
+version = "0.23.4"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"

port_ocean-0.23.3/port_ocean/helpers/metric/utils.py

@@ -1,28 +0,0 @@
-from functools import wraps
-import time
-from typing import Any, Callable
-
-from port_ocean.context.ocean import ocean
-from port_ocean.helpers.metric.metric import MetricType
-
-
-def TimeMetric(phase: str) -> Any:
-    def decorator(func: Callable[..., Any]) -> Any:
-
-        @wraps(func)
-        async def wrapper(*args: Any, **kwargs: dict[Any, Any]) -> Any:
-            start = time.monotonic()
-            res = await func(*args, **kwargs)
-            end = time.monotonic()
-            duration = end - start
-            ocean.metrics.inc_metric(
-                name=MetricType.DURATION_NAME,
-                labels=[ocean.metrics.current_resource_kind(), phase],
-                value=duration,
-            )
-
-            return res
-
-        return wrapper
-
-    return decorator