port-ocean 0.22.8__py3-none-any.whl → 0.22.10__py3-none-any.whl

port_ocean/clients/port/mixins/entities.py

@@ -5,7 +5,7 @@ from urllib.parse import quote_plus
 
 import httpx
 from loguru import logger
-
+from port_ocean.context.ocean import ocean
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.types import RequestOptions, UserAgentType
 from port_ocean.clients.port.utils import (
@@ -15,6 +15,8 @@ from port_ocean.clients.port.utils import (
 from port_ocean.core.models import Entity, PortAPIErrorMessage
 from starlette import status
 
+from port_ocean.helpers.metric.metric import MetricPhase, MetricType
+
 
 class EntityClientMixin:
     def __init__(self, auth: PortAuthentication, client: httpx.AsyncClient):
@@ -81,6 +83,15 @@ class EntityClientMixin:
                 f"blueprint: {entity.blueprint}"
             )
             result = response.json()
+            ocean.metrics.inc_metric(
+                name=MetricType.OBJECT_COUNT_NAME,
+                labels=[
+                    ocean.metrics.current_resource_kind(),
+                    MetricPhase.LOAD,
+                    MetricPhase.LoadResult.FAILED,
+                ],
+                value=1,
+            )
 
             if (
                 response.status_code == status.HTTP_404_NOT_FOUND
@@ -89,6 +100,17 @@ class EntityClientMixin:
             ):
                 # Return false to differentiate from `result_entity.is_using_search_identifier`
                 return False
+        else:
+            ocean.metrics.inc_metric(
+                name=MetricType.OBJECT_COUNT_NAME,
+                labels=[
+                    ocean.metrics.current_resource_kind(),
+                    MetricPhase.LOAD,
+                    MetricPhase.LoadResult.LOADED,
+                ],
+                value=1,
+            )
+
         handle_port_status_code(response, should_raise)
         result = response.json()
 

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -103,7 +103,7 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
             and deletion_rate <= entity_deletion_threshold
         ):
             await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
-            ocean.metrics.set_metric(
+            ocean.metrics.inc_metric(
                 name=MetricType.DELETION_COUNT_NAME,
                 labels=[ocean.metrics.current_resource_kind(), MetricPhase.DELETE],
                 value=len(diff.deleted),

port_ocean/core/integrations/mixins/sync_raw.py

@@ -31,7 +31,7 @@ from port_ocean.core.ocean_types import (
 )
 from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
-from port_ocean.helpers.metric.metric import MetricType, MetricPhase
+from port_ocean.helpers.metric.metric import SyncState, MetricType, MetricPhase
 from port_ocean.helpers.metric.utils import TimeMetric
 
 SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
@@ -238,11 +238,21 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 if changed_entities:
                     logger.info("Upserting changed entities", changed_entities=len(changed_entities),
                         total_entities=len(objects_diff[0].entity_selector_diff.passed))
+                    ocean.metrics.inc_metric(
+                            name=MetricType.OBJECT_COUNT_NAME,
+                            labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.SKIPPED],
+                            value=len(objects_diff[0].entity_selector_diff.passed) - len(changed_entities)
+                        )
                     await self.entities_state_applier.upsert(
                         changed_entities, user_agent_type
                     )
                 else:
                     logger.info("Entities in batch didn't changed since last sync, skipping", total_entities=len(objects_diff[0].entity_selector_diff.passed))
+                    ocean.metrics.inc_metric(
+                            name=MetricType.OBJECT_COUNT_NAME,
+                            labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD, MetricPhase.LoadResult.SKIPPED],
+                            value=len(objects_diff[0].entity_selector_diff.passed)
+                        )
                 modified_objects = [ocean.port_client._reduce_entity(entity) for entity in objects_diff[0].entity_selector_diff.passed]
             except Exception as e:
                 logger.warning(f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}")
@@ -335,6 +345,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                     passed_entities.extend(calculation_result.entity_selector_diff.passed)
                     number_of_transformed_entities += calculation_result.number_of_transformed_entities
             except* OceanAbortException as error:
+                ocean.metrics.sync_state = SyncState.FAILED
                 errors.append(error)
 
         logger.info(
@@ -347,18 +358,30 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             value=int(not errors)
         )
 
-        ocean.metrics.set_metric(
+        ocean.metrics.inc_metric(
             name=MetricType.OBJECT_COUNT_NAME,
-            labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT],
+            labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT , MetricPhase.ExtractResult.EXTRACTED],
             value=number_of_raw_results
         )
 
-        ocean.metrics.set_metric(
+        ocean.metrics.inc_metric(
             name=MetricType.OBJECT_COUNT_NAME,
-            labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM],
+            labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM , MetricPhase.TransformResult.TRANSFORMED],
             value=number_of_transformed_entities
         )
 
+        ocean.metrics.inc_metric(
+            name=MetricType.OBJECT_COUNT_NAME,
+            labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM , MetricPhase.TransformResult.FILTERED_OUT],
+            value=number_of_raw_results -number_of_transformed_entities
+        )
+
+        ocean.metrics.inc_metric(
+            name=MetricType.OBJECT_COUNT_NAME,
+            labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM , MetricPhase.TransformResult.FAILED],
+            value=len(errors)
+        )
+
         return passed_entities, errors
 
     async def register_raw(
@@ -558,7 +581,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 for entity in event.entity_topological_sorter.get_entities(False):
                     await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
 
-
     @TimeMetric(MetricPhase.RESYNC)
     async def sync_raw_all(
         self,
@@ -584,12 +606,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             EventType.RESYNC,
             trigger_type=trigger_type,
         ):
+            ocean.metrics.event_id = event.id
+
             # If a resync is triggered due to a mappings change, we want to make sure that we have the updated version
             # rather than the old cache
             app_config = await self.port_app_config_handler.get_port_app_config(
                 use_cache=False
             )
             logger.info(f"Resync will use the following mappings: {app_config.dict()}")
+            ocean.metrics.initialize_metrics([f"{resource.kind}-{index}" for index, resource in enumerate(app_config.resources)])
+            await ocean.metrics.flush()
 
             # Execute resync_start hooks
             for resync_start_fn in self.event_strategy["resync_start"]:
@@ -616,20 +642,19 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                     # config as we might have multiple resources in the same event
                     async with resource_context(resource,index):
                         resource_kind_id = f"{resource.kind}-{index}"
+                        ocean.metrics.sync_state = SyncState.SYNCING
+                        await ocean.metrics.flush(kind=resource_kind_id)
+
                         task = asyncio.create_task(
                             self._register_in_batches(resource, user_agent_type)
                         )
 
                         event.on_abort(lambda: task.cancel())
                         kind_results: tuple[list[Entity], list[Exception]] = await task
-                        ocean.metrics.set_metric(
-                            name=MetricType.OBJECT_COUNT_NAME,
-                            labels=[ocean.metrics.current_resource_kind(), MetricPhase.LOAD],
-                            value=len(kind_results[0])
-                        )
 
                         creation_results.append(kind_results)
-
+                        if ocean.metrics.sync_state != SyncState.FAILED:
+                            ocean.metrics.sync_state = SyncState.COMPLETED
                         await ocean.metrics.flush(kind=resource_kind_id)
 
                 await self.sort_and_upsert_failed_entities(user_agent_type)

port_ocean/helpers/metric/metric.py

@@ -22,6 +22,19 @@ class MetricPhase:
     RESYNC = "resync"
     DELETE = "delete"
 
+    class TransformResult:
+        TRANSFORMED = "transformed"
+        FILTERED_OUT = "filtered_out"
+        FAILED = "failed"
+
+    class LoadResult:
+        LOADED = "loaded"
+        FAILED = "failed"
+        SKIPPED = "skipped"
+
+    class ExtractResult:
+        EXTRACTED = "raw_extracted"
+
 
 class MetricType:
     # Define metric names as constants
@@ -33,6 +46,13 @@ class MetricType:
     DELETION_COUNT_NAME = "deletion_count"
 
 
+class SyncState:
+    SYNCING = "syncing"
+    COMPLETED = "completed"
+    QUEUED = "queued"
+    FAILED = "failed"
+
+
 # Registry for core and custom metrics
 _metrics_registry: Dict[str, Tuple[str, str, List[str]]] = {
     MetricType.DURATION_NAME: (
@@ -43,7 +63,7 @@ _metrics_registry: Dict[str, Tuple[str, str, List[str]]] = {
     MetricType.OBJECT_COUNT_NAME: (
         MetricType.OBJECT_COUNT_NAME,
         "object_count description",
-        ["kind", "phase"],
+        ["kind", "phase", "object_count_type"],
     ),
     MetricType.ERROR_COUNT_NAME: (
         MetricType.ERROR_COUNT_NAME,
@@ -86,6 +106,9 @@ class EmptyMetric:
     def labels(self, *args: Any) -> None:
         return None
 
+    def inc(self, *args: Any) -> None:
+        return None
+
 
 class Metrics:
     def __init__(
@@ -100,6 +123,24 @@ class Metrics:
         self.load_metrics()
         self._integration_version: Optional[str] = None
         self._ocean_version: Optional[str] = None
+        self.event_id = ""
+        self.sync_state = SyncState.QUEUED
+
+    @property
+    def event_id(self) -> str:
+        return self._event_id
+
+    @event_id.setter
+    def event_id(self, value: str) -> None:
+        self._event_id = value
+
+    @property
+    def sync_state(self) -> str:
+        return self._sync_state
+
+    @sync_state.setter
+    def sync_state(self, value: str) -> None:
+        self._sync_state = value
 
     @property
     def integration_version(self) -> str:
@@ -139,6 +180,19 @@ class Metrics:
             return EmptyMetric()
         return metrics.labels(*labels)
 
+    def inc_metric(self, name: str, labels: list[str], value: float) -> None:
+        """Increment a metric value in a single method call.
+
+        Args:
+            name (str): The metric name to inc.
+            labels (list[str]): The labels to apply to the metric.
+            value (float): The value to inc.
+        """
+        if not self.enabled:
+            return None
+
+        self.get_metric(name, labels).inc(value)
+
     def set_metric(self, name: str, labels: list[str], value: float) -> None:
         """Set a metric value in a single method call.
 
@@ -152,6 +206,49 @@ class Metrics:
 
         self.get_metric(name, labels).set(value)
 
+    def initialize_metrics(self, kind_blockes: list[str]) -> None:
+        for kind in kind_blockes:
+            self.set_metric(MetricType.SUCCESS_NAME, [kind, MetricPhase.RESYNC], 0)
+            self.set_metric(MetricType.DURATION_NAME, [kind, MetricPhase.RESYNC], 0)
+
+            self.set_metric(
+                MetricType.OBJECT_COUNT_NAME,
+                [kind, MetricPhase.EXTRACT, MetricPhase.ExtractResult.EXTRACTED],
+                0,
+            )
+
+            self.set_metric(
+                MetricType.OBJECT_COUNT_NAME,
+                [kind, MetricPhase.TRANSFORM, MetricPhase.TransformResult.TRANSFORMED],
+                0,
+            )
+            self.set_metric(
+                MetricType.OBJECT_COUNT_NAME,
+                [kind, MetricPhase.TRANSFORM, MetricPhase.TransformResult.FILTERED_OUT],
+                0,
+            )
+            self.set_metric(
+                MetricType.OBJECT_COUNT_NAME,
+                [kind, MetricPhase.TRANSFORM, MetricPhase.TransformResult.FAILED],
+                0,
+            )
+
+            self.set_metric(
+                MetricType.OBJECT_COUNT_NAME,
+                [kind, MetricPhase.LOAD, MetricPhase.LoadResult.LOADED],
+                0,
+            )
+            self.set_metric(
+                MetricType.OBJECT_COUNT_NAME,
+                [kind, MetricPhase.LOAD, MetricPhase.LoadResult.FAILED],
+                0,
+            )
+            self.set_metric(
+                MetricType.OBJECT_COUNT_NAME,
+                [kind, MetricPhase.LOAD, MetricPhase.LoadResult.SKIPPED],
+                0,
+            )
+
     def create_mertic_router(self) -> APIRouter:
         if not self.enabled:
             return APIRouter()
@@ -201,14 +298,21 @@ class Metrics:
                         if kind and sample.labels.get("kind") != kind:
                             continue
 
-                        # Create nested dictionary structure based on labels
-                        for key, value in sample.labels.items():
-                            if key not in current_level:
-                                current_level[key] = {}
-                            current_level = current_level[key]
-                            if value not in current_level:
-                                current_level[value] = {}
-                            current_level = current_level[value]
+                        # Get the ordered labels from the registry
+                        ordered_labels = _metrics_registry.get(
+                            sample.name, (None, None, [])
+                        )[2]
+
+                        # Create nested dictionary structure based on ordered labels
+                        for label_name in ordered_labels:
+                            if label_name in sample.labels:
+                                value = sample.labels[label_name]
+                                if label_name not in current_level:
+                                    current_level[label_name] = {}
+                                current_level = current_level[label_name]
+                                if value not in current_level:
+                                    current_level[value] = {}
+                                current_level = current_level[value]
 
                     current_level[sample.name] = sample.value
 
@@ -228,6 +332,8 @@ class Metrics:
                     "ocean_version": self.ocean_version,
                     "kind_identifier": kind_key,
                     "kind": "-".join(kind_key.split("-")[:-1]),
+                    "event_id": self.event_id,
+                    "sync_state": self.sync_state,
                     "metrics": metrics,
                 }
                 logger.info(f"Sending metrics to webhook {kind_key}: {event}")

port_ocean/helpers/metric/utils.py

@@ -17,7 +17,7 @@ def TimeMetric(phase: str) -> Any:
             res = await func(*args, **kwargs)
             end = time.monotonic()
             duration = end - start
-            ocean.metrics.set_metric(
+            ocean.metrics.inc_metric(
                 name=MetricType.DURATION_NAME,
                 labels=[ocean.metrics.current_resource_kind(), phase],
                 value=duration,

{port_ocean-0.22.8.dist-info → port_ocean-0.22.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.22.8
+Version: 0.22.10
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.22.8.dist-info → port_ocean-0.22.10.dist-info}/RECORD

@@ -52,7 +52,7 @@ port_ocean/clients/port/authentication.py,sha256=r7r8Ag9WuwXy-CmgeOoj-PHbmJAQxhb
 port_ocean/clients/port/client.py,sha256=dv0mxIOde6J-wFi1FXXZkoNPVHrZzY7RSMhNkDD9xgA,3566
 port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/mixins/blueprints.py,sha256=aMCG4zePsMSMjMLiGrU37h5z5_ElfMzTcTvqvOI5wXY,4683
-port_ocean/clients/port/mixins/entities.py,sha256=MaJjkp16wxc2fkd2BwgVIhHU05fi-7pM9OgX_cV3VmA,10901
+port_ocean/clients/port/mixins/entities.py,sha256=-1Gs74z_8eviWItHIpveQhKdA7gnjbqZ3STS4jgGONs,11668
 port_ocean/clients/port/mixins/integrations.py,sha256=tvuP8jxgbDyTwGfp8hd0-NXfnsMjZFJXDEDzXroctnY,9381
 port_ocean/clients/port/mixins/migrations.py,sha256=vdL_A_NNUogvzujyaRLIoZEu5vmKDY2BxTjoGP94YzI,1467
 port_ocean/clients/port/mixins/organization.py,sha256=A2cP5V49KnjoAXxjmnm_XGth4ftPSU0qURNfnyUyS_Y,1041
@@ -87,7 +87,7 @@ port_ocean/core/handlers/base.py,sha256=cTarblazu8yh8xz2FpB-dzDKuXxtoi143XJgPbV_
 port_ocean/core/handlers/entities_state_applier/__init__.py,sha256=kgLZDCeCEzi4r-0nzW9k78haOZNf6PX7mJOUr34A4c8,173
 port_ocean/core/handlers/entities_state_applier/base.py,sha256=5wHL0icfFAYRPqk8iV_wN49GdJ3aRUtO8tumSxBi4Wo,2268
 port_ocean/core/handlers/entities_state_applier/port/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=CcYSTMa4nA-fDX_9ZswthoyK8jMkupS1v-LMB-XxYBk,6254
+port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=Glw-iSZvN0WaaQL3qfOSb9uEXcy88_mSOzifMLNhhOc,6254
 port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha256=1zncwCbE-Gej0xaWKlzZgoXxOBe9bgs_YxlZ8QW3NdI,1751
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
@@ -113,7 +113,7 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=8HklZmlyffYY_LeDe8xbt3Tb08rlLkqVhFF-2NQeJP4,4126
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=uATzik6gSmkpKlIhbcRavlDo-xtkUvMwNp1wZltJj-I,27487
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=ezjfpxZJ-0IjxppP6nKu4lSWjsh_8HPNghSgsQRayR8,29100
 port_ocean/core/integrations/mixins/utils.py,sha256=oN4Okz6xlaefpid1_Pud8HPSw9BwwjRohyNsknq-Myg,2309
 port_ocean/core/models.py,sha256=YpJ2XOB3Zt9_M-rcMrMjugFNzBDg2hCUKgqvEt7now0,2348
 port_ocean/core/ocean_types.py,sha256=4VipWFOHEh_d9LmWewQccwx1p2dtrRYW0YURVgNsAjo,1398
@@ -131,8 +131,8 @@ port_ocean/exceptions/utils.py,sha256=gjOqpi-HpY1l4WlMFsGA9yzhxDhajhoGGdDDyGbLnq
 port_ocean/exceptions/webhook_processor.py,sha256=yQYazg53Y-ohb7HfViwq1opH_ZUuUdhHSRxcUNveFpI,114
 port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/helpers/async_client.py,sha256=SRlP6o7_FCSY3UHnRlZdezppePVxxOzZ0z861vE3K40,1783
-port_ocean/helpers/metric/metric.py,sha256=EQstAxlsC-lsp8VFw4GgxCh3SGV9t2Y-j1Z1qbXyy-I,8014
-port_ocean/helpers/metric/utils.py,sha256=1DlLR9FmUOpSlWSfY_tUEGcxRQY6ckAfbD3dnMnxd4c,874
+port_ocean/helpers/metric/metric.py,sha256=Eg8t7k3hDumnUHe2TvE5xus3cWoROwJPjoviW4QrGoE,11440
+port_ocean/helpers/metric/utils.py,sha256=_qj9-PfsrMHoXVscEsIdnylLVZJnupg8ofHYaKc5-t8,874
 port_ocean/helpers/retry.py,sha256=gmS4YxM6N4fboFp7GSgtOzyBJemxs46bnrz4L4rDS6Y,16136
 port_ocean/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/log/handlers.py,sha256=ncVjgqrZRh6BhyRrA6DQG86Wsbxph1yWYuEC0cWfe-Q,3631
@@ -188,8 +188,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.22.8.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.22.8.dist-info/METADATA,sha256=o8OYwTwdxk24xZt7KB8kXJLN_ukZNrnu3lMZOPTlbW8,6764
-port_ocean-0.22.8.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.22.8.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.22.8.dist-info/RECORD,,
+port_ocean-0.22.10.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.22.10.dist-info/METADATA,sha256=ixe0DL4U9Vy7fbJ7HmdFpkht_yjFwJL5ANw8ZahOv2U,6765
+port_ocean-0.22.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.22.10.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.22.10.dist-info/RECORD,,