port-ocean 0.22.10__tar.gz → 0.22.11__tar.gz

{port_ocean-0.22.10 → port_ocean-0.22.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.22.10
+Version: 0.22.11
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.22.10 → port_ocean-0.22.11}/port_ocean/clients/port/mixins/integrations.py

@@ -25,6 +25,10 @@ class LogAttributes(TypedDict):
     ingestUrl: str
 
 
+class MetricsAttributes(TypedDict):
+    ingestUrl: str
+
+
 class IntegrationClientMixin:
     def __init__(
         self,
@@ -38,6 +42,7 @@ class IntegrationClientMixin:
         self.auth = auth
         self.client = client
         self._log_attributes: LogAttributes | None = None
+        self._metrics_attributes: MetricsAttributes | None = None
 
     async def is_integration_provision_enabled(
         self, integration_type: str, should_raise: bool = True, should_log: bool = True
@@ -106,6 +111,12 @@ class IntegrationClientMixin:
             self._log_attributes = response["logAttributes"]
         return self._log_attributes
 
+    async def get_metrics_attributes(self) -> LogAttributes:
+        if self._metrics_attributes is None:
+            response = await self.get_current_integration()
+            self._metrics_attributes = response["metricAttributes"]
+        return self._metrics_attributes
+
     async def _poll_integration_until_default_provisioning_is_complete(
         self,
     ) -> Dict[str, Any]:
@@ -197,6 +208,40 @@ class IntegrationClientMixin:
         handle_port_status_code(response)
         return response.json()["integration"]
 
+    async def post_integration_sync_metrics(
+        self, metrics: list[dict[str, Any]]
+    ) -> None:
+        logger.debug("starting POST metrics request", metrics=metrics)
+        metrics_attributes = await self.get_metrics_attributes()
+        headers = await self.auth.headers()
+        response = await self.client.post(
+            metrics_attributes["ingestUrl"],
+            headers=headers,
+            json={
+                "syncKindsMetrics": metrics,
+            },
+        )
+        handle_port_status_code(response, should_log=False)
+        logger.debug("Finished POST metrics request")
+
+    async def put_integration_sync_metrics(self, kind_metrics: dict[str, Any]) -> None:
+        logger.debug("starting PUT metrics request", kind_metrics=kind_metrics)
+        metrics_attributes = await self.get_metrics_attributes()
+        url = (
+            metrics_attributes["ingestUrl"]
+            + f"/resync/{kind_metrics['eventId']}/kind/{kind_metrics['kindIdentifier']}"
+        )
+        headers = await self.auth.headers()
+        response = await self.client.put(
+            url,
+            headers=headers,
+            json={
+                "syncKindMetrics": kind_metrics,
+            },
+        )
+        handle_port_status_code(response, should_log=False)
+        logger.debug("Finished PUT metrics request")
+
     async def ingest_integration_logs(self, logs: list[dict[str, Any]]) -> None:
         logger.debug("Ingesting logs")
         log_attributes = await self.get_log_attributes()

{port_ocean-0.22.10 → port_ocean-0.22.11}/port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -104,8 +104,12 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         ):
             await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
             ocean.metrics.inc_metric(
-                name=MetricType.DELETION_COUNT_NAME,
-                labels=[ocean.metrics.current_resource_kind(), MetricPhase.DELETE],
+                name=MetricType.OBJECT_COUNT_NAME,
+                labels=[
+                    ocean.metrics.current_resource_kind(),
+                    MetricPhase.DELETE,
+                    MetricPhase.DeletionResult.DELETED,
+                ],
                 value=len(diff.deleted),
             )
         else:

{port_ocean-0.22.10 → port_ocean-0.22.11}/port_ocean/core/handlers/resync_state_updater/updater.py

@@ -90,4 +90,10 @@ class ResyncStateUpdater:
             labels=[ocean.metrics.current_resource_kind(), MetricPhase.RESYNC],
             value=int(status == IntegrationStateStatus.Completed),
         )
-        await ocean.metrics.flush(kind=ocean.metrics.current_resource_kind())
+
+        await ocean.metrics.send_metrics_to_webhook(
+            kind=ocean.metrics.current_resource_kind()
+        )
+        # await ocean.metrics.report_sync_metrics(
+        #     kinds=[ocean.metrics.current_resource_kind()]
+        # ) # TODO: uncomment this when end points are ready

{port_ocean-0.22.10 → port_ocean-0.22.11}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -614,8 +614,10 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 use_cache=False
             )
             logger.info(f"Resync will use the following mappings: {app_config.dict()}")
-            ocean.metrics.initialize_metrics([f"{resource.kind}-{index}" for index, resource in enumerate(app_config.resources)])
-            await ocean.metrics.flush()
+
+            kinds = [f"{resource.kind}-{index}" for index, resource in enumerate(app_config.resources)]
+            ocean.metrics.initialize_metrics(kinds)
+            # await ocean.metrics.report_sync_metrics(kinds=kinds) # TODO: uncomment this when end points are ready
 
             # Execute resync_start hooks
             for resync_start_fn in self.event_strategy["resync_start"]:
@@ -643,7 +645,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                     async with resource_context(resource,index):
                         resource_kind_id = f"{resource.kind}-{index}"
                         ocean.metrics.sync_state = SyncState.SYNCING
-                        await ocean.metrics.flush(kind=resource_kind_id)
+                        # await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
 
                         task = asyncio.create_task(
                             self._register_in_batches(resource, user_agent_type)
@@ -653,9 +655,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                         kind_results: tuple[list[Entity], list[Exception]] = await task
 
                         creation_results.append(kind_results)
+
                         if ocean.metrics.sync_state != SyncState.FAILED:
                             ocean.metrics.sync_state = SyncState.COMPLETED
-                        await ocean.metrics.flush(kind=resource_kind_id)
+
+                        await ocean.metrics.send_metrics_to_webhook(
+                            kind=resource_kind_id
+                        )
+                        # await ocean.metrics.report_kind_sync_metrics(kind=resource_kind_id) # TODO: uncomment this when end points are ready
 
                 await self.sort_and_upsert_failed_entities(user_agent_type)
 

{port_ocean-0.22.10 → port_ocean-0.22.11}/port_ocean/helpers/metric/metric.py

@@ -13,6 +13,7 @@ import prometheus_client.parser
 
 if TYPE_CHECKING:
     from port_ocean.config.settings import MetricsSettings, IntegrationSettings
+    from port_ocean.clients.port.client import PortClient
 
 
 class MetricPhase:
@@ -35,21 +36,22 @@ class MetricPhase:
     class ExtractResult:
         EXTRACTED = "raw_extracted"
 
+    class DeletionResult:
+        DELETED = "deleted"
+
 
 class MetricType:
     # Define metric names as constants
     DURATION_NAME = "duration_seconds"
     OBJECT_COUNT_NAME = "object_count"
-    ERROR_COUNT_NAME = "error_count"
     SUCCESS_NAME = "success"
     RATE_LIMIT_WAIT_NAME = "rate_limit_wait_seconds"
-    DELETION_COUNT_NAME = "deletion_count"
 
 
 class SyncState:
     SYNCING = "syncing"
     COMPLETED = "completed"
-    QUEUED = "queued"
+    PENDING = "pending"
     FAILED = "failed"
 
 
@@ -65,11 +67,6 @@ _metrics_registry: Dict[str, Tuple[str, str, List[str]]] = {
         "object_count description",
         ["kind", "phase", "object_count_type"],
     ),
-    MetricType.ERROR_COUNT_NAME: (
-        MetricType.ERROR_COUNT_NAME,
-        "error_count description",
-        ["kind", "phase"],
-    ),
     MetricType.SUCCESS_NAME: (
         MetricType.SUCCESS_NAME,
         "success description",
@@ -80,11 +77,6 @@ _metrics_registry: Dict[str, Tuple[str, str, List[str]]] = {
         "rate_limit_wait description",
         ["kind", "phase", "endpoint"],
     ),
-    MetricType.DELETION_COUNT_NAME: (
-        MetricType.DELETION_COUNT_NAME,
-        "deletion_count description",
-        ["kind", "phase"],
-    ),
 }
 
 
@@ -115,16 +107,18 @@ class Metrics:
         self,
         metrics_settings: "MetricsSettings",
         integration_configuration: "IntegrationSettings",
+        port_client: "PortClient",
     ) -> None:
         self.metrics_settings = metrics_settings
         self.integration_configuration = integration_configuration
+        self.port_client = port_client
         self.registry = prometheus_client.CollectorRegistry()
         self.metrics: dict[str, Gauge] = {}
         self.load_metrics()
         self._integration_version: Optional[str] = None
         self._ocean_version: Optional[str] = None
         self.event_id = ""
-        self.sync_state = SyncState.QUEUED
+        self.sync_state = SyncState.PENDING
 
     @property
     def event_id(self) -> str:
@@ -163,9 +157,6 @@ class Metrics:
         return self.metrics_settings.enabled
 
     def load_metrics(self) -> None:
-        if not self.enabled:
-            return None
-
         # Load all registered metrics
         for name, (_, description, labels) in _metrics_registry.items():
             self.metrics[name] = Gauge(
@@ -173,8 +164,6 @@ class Metrics:
             )
 
     def get_metric(self, name: str, labels: list[str]) -> Gauge | EmptyMetric:
-        if not self.enabled:
-            return EmptyMetric()
         metrics = self.metrics.get(name)
         if not metrics:
             return EmptyMetric()
@@ -188,9 +177,6 @@ class Metrics:
             labels (list[str]): The labels to apply to the metric.
             value (float): The value to inc.
         """
-        if not self.enabled:
-            return None
-
         self.get_metric(name, labels).inc(value)
 
     def set_metric(self, name: str, labels: list[str], value: float) -> None:
@@ -201,9 +187,6 @@ class Metrics:
             labels (list[str]): The labels to apply to the metric.
             value (float): The value to set.
         """
-        if not self.enabled:
-            return None
-
         self.get_metric(name, labels).set(value)
 
     def initialize_metrics(self, kind_blockes: list[str]) -> None:
@@ -271,15 +254,39 @@ class Metrics:
             self.registry
         ).decode()
 
-    async def flush(
-        self, metric_name: Optional[str] = None, kind: Optional[str] = None
+    async def report_sync_metrics(
+        self, metric_name: Optional[str] = None, kinds: Optional[list[str]] = None
     ) -> None:
-        if not self.enabled:
+        if kinds is None:
             return None
 
-        if not self.metrics_settings.webhook_url:
+        metrics = []
+
+        for kind in kinds:
+            metric = self.generate_metrics(metric_name, kind)
+            metrics.extend(metric)
+
+        try:
+            await self.port_client.post_integration_sync_metrics(metrics)
+        except Exception as e:
+            logger.error(f"Error posting metrics: {e}", metrics=metrics)
+
+    async def report_kind_sync_metrics(
+        self, metric_name: Optional[str] = None, kind: Optional[str] = None
+    ) -> None:
+        metrics = self.generate_metrics(metric_name, kind)
+        if not metrics:
             return None
 
+        try:
+            for metric in metrics:
+                await self.port_client.put_integration_sync_metrics(metric)
+        except Exception as e:
+            logger.error(f"Error putting metrics: {e}", metrics=metrics)
+
+    def generate_metrics(
+        self, metric_name: Optional[str] = None, kind: Optional[str] = None
+    ) -> list[dict[str, Any]]:
         try:
             latest_raw = self.generate_latest()
             metric_families = prometheus_client.parser.text_string_to_metric_families(
@@ -318,27 +325,54 @@ class Metrics:
 
             # If no metrics were filtered, exit early
             if not metrics_dict.get("kind", {}):
-                return None
+                return []
 
+            events = []
             for kind_key, metrics in metrics_dict.get("kind", {}).items():
                 # Skip if we're filtering by kind and this isn't the requested kind
                 if kind and kind_key != kind:
                     continue
 
                 event = {
-                    "integration_type": self.integration_configuration.type,
-                    "integration_identifier": self.integration_configuration.identifier,
-                    "integration_version": self.integration_version,
-                    "ocean_version": self.ocean_version,
-                    "kind_identifier": kind_key,
-                    "kind": "-".join(kind_key.split("-")[:-1]),
-                    "event_id": self.event_id,
-                    "sync_state": self.sync_state,
+                    "integrationType": self.integration_configuration.type,
+                    "integrationIdentifier": self.integration_configuration.identifier,
+                    "integrationVersion": self.integration_version,
+                    "oceanVersion": self.ocean_version,
+                    "kindIdentifier": kind_key,
+                    "kind": (
+                        "-".join(kind_key.split("-")[:-1])
+                        if "-" in kind_key
+                        else kind_key
+                    ),
+                    "kindIndex": 0 if kind_key == "__runtime__" else int(kind_key[-1]),
+                    "eventId": self.event_id,
+                    "syncState": self.sync_state,
                     "metrics": metrics,
                 }
-                logger.info(f"Sending metrics to webhook {kind_key}: {event}")
+                events.append(event)
+            return events
+        except Exception as e:
+            logger.error(f"Error sending metrics to webhook: {e}")
+            return []
+
+    async def send_metrics_to_webhook(
+        self, metric_name: Optional[str] = None, kind: Optional[str] = None
+    ) -> None:
+        try:
+            if not self.enabled:
+                return None
+
+            if not self.metrics_settings.webhook_url:
+                return None
+
+            metrics = self.generate_metrics(metric_name, kind)
+            if not metrics:
+                return None
+
+            for metric in metrics:
+                logger.info(f"Sending metrics to webhook {metric['kind']}: {metric}")
                 await AsyncClient().post(
-                    url=self.metrics_settings.webhook_url, json=event
+                    url=self.metrics_settings.webhook_url, json=metric
                 )
         except Exception as e:
             logger.error(f"Error sending metrics to webhook: {e}")

{port_ocean-0.22.10 → port_ocean-0.22.11}/port_ocean/helpers/metric/utils.py

@@ -11,8 +11,6 @@ def TimeMetric(phase: str) -> Any:
 
         @wraps(func)
         async def wrapper(*args: Any, **kwargs: dict[Any, Any]) -> Any:
-            if not ocean.metrics.enabled:
-                return await func(*args, **kwargs)
             start = time.monotonic()
             res = await func(*args, **kwargs)
             end = time.monotonic()

{port_ocean-0.22.10 → port_ocean-0.22.11}/port_ocean/ocean.py

@@ -57,9 +57,20 @@ class Ocean:
             *self.config.get_sensitive_fields_data()
         )
         self.integration_router = integration_router or APIRouter()
+
+        self.port_client = PortClient(
+            base_url=self.config.port.base_url,
+            client_id=self.config.port.client_id,
+            client_secret=self.config.port.client_secret,
+            integration_identifier=self.config.integration.identifier,
+            integration_type=self.config.integration.type,
+            integration_version=__integration_version__,
+        )
+
         self.metrics = port_ocean.helpers.metric.metric.Metrics(
             metrics_settings=self.config.metrics,
             integration_configuration=self.config.integration,
+            port_client=self.port_client,
         )
 
         self.webhook_manager = LiveEventsProcessorManager(
@@ -69,14 +80,6 @@ class Ocean:
             max_wait_seconds_before_shutdown=self.config.max_wait_seconds_before_shutdown,
         )
 
-        self.port_client = PortClient(
-            base_url=self.config.port.base_url,
-            client_id=self.config.port.client_id,
-            client_secret=self.config.port.client_secret,
-            integration_identifier=self.config.integration.identifier,
-            integration_type=self.config.integration.type,
-            integration_version=__integration_version__,
-        )
         self.integration = (
             integration_class(ocean) if integration_class else BaseIntegration(ocean)
         )