port-ocean 0.24.2__py3-none-any.whl → 0.24.4__py3-none-any.whl

port_ocean/clients/port/mixins/entities.py

@@ -353,66 +353,60 @@ class EntityClientMixin:
         entities_results: list[tuple[bool, Entity]] = []
         blueprint = entities[0].blueprint
 
-        if ocean.app.is_saas():
-            bulk_size = self.calculate_entities_batch_size(entities)
-            bulks = [
-                entities[i : i + bulk_size] for i in range(0, len(entities), bulk_size)
-            ]
-
-            bulk_results = await asyncio.gather(
-                *(
-                    self.upsert_entities_bulk(
-                        blueprint,
-                        bulk,
-                        request_options,
-                        user_agent_type,
-                        should_raise=should_raise,
-                    )
-                    for bulk in bulks
-                ),
-                return_exceptions=True,
-            )
+        bulk_size = self.calculate_entities_batch_size(entities)
+        bulks = [
+            entities[i : i + bulk_size] for i in range(0, len(entities), bulk_size)
+        ]
+
+        bulk_results = await asyncio.gather(
+            *(
+                self.upsert_entities_bulk(
+                    blueprint,
+                    bulk,
+                    request_options,
+                    user_agent_type,
+                    should_raise=should_raise,
+                )
+                for bulk in bulks
+            ),
+            return_exceptions=True,
+        )
 
-            for bulk, bulk_result in zip(bulks, bulk_results):
-                if isinstance(bulk_result, httpx.HTTPStatusError) or isinstance(
-                    bulk_result, Exception
+        for bulk, bulk_result in zip(bulks, bulk_results):
+            if isinstance(bulk_result, httpx.HTTPStatusError) or isinstance(
+                bulk_result, Exception
+            ):
+                if should_raise:
+                    raise bulk_result
+                # If should_raise is False, retry batch in sequential order as a fallback only for 413 errors
+                if (
+                    isinstance(bulk_result, httpx.HTTPStatusError)
+                    and bulk_result.response.status_code == 413
                 ):
-                    if should_raise:
-                        raise bulk_result
-                    # If should_raise is False, retry batch in sequential order as a fallback only for 413 errors
+                    individual_upsert_results = (
+                        await self._upsert_entities_batch_individually(
+                            bulk, request_options, user_agent_type, should_raise
+                        )
+                    )
+                    entities_results.extend(individual_upsert_results)
+                else:
+                    # For other errors, mark all entities in the batch as failed
+                    for entity in bulk:
+                        failed_result: tuple[bool, Entity] = (
+                            False,
+                            self._reduce_entity(entity),
+                        )
+                        entities_results.append(failed_result)
+            elif isinstance(bulk_result, list):
+                for status, entity in bulk_result:
                     if (
-                        isinstance(bulk_result, httpx.HTTPStatusError)
-                        and bulk_result.response.status_code == 413
-                    ):
-                        individual_upsert_results = (
-                            await self._upsert_entities_batch_individually(
-                                bulk, request_options, user_agent_type, should_raise
-                            )
+                        status is not None
+                    ):  # when using the search identifier we might not have an actual identifier
+                        bulk_result_tuple: tuple[bool, Entity] = (
+                            bool(status),
+                            entity,
                         )
-                        entities_results.extend(individual_upsert_results)
-                    else:
-                        # For other errors, mark all entities in the batch as failed
-                        for entity in bulk:
-                            failed_result: tuple[bool, Entity] = (
-                                False,
-                                self._reduce_entity(entity),
-                            )
-                            entities_results.append(failed_result)
-                elif isinstance(bulk_result, list):
-                    for status, entity in bulk_result:
-                        if (
-                            status is not None
-                        ):  # when using the search identifier we might not have an actual identifier
-                            bulk_result_tuple: tuple[bool, Entity] = (
-                                bool(status),
-                                entity,
-                            )
-                            entities_results.append(bulk_result_tuple)
-        else:
-            individual_upsert_results = await self._upsert_entities_batch_individually(
-                entities, request_options, user_agent_type, should_raise
-            )
-            entities_results.extend(individual_upsert_results)
+                        entities_results.append(bulk_result_tuple)
 
         return entities_results
 

port_ocean/config/settings.py

@@ -108,7 +108,6 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
 
     upsert_entities_batch_max_length: int = 20
     upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
-    bulk_upserts_enabled: bool = False
 
     @validator("process_execution_mode")
     def validate_process_execution_mode(

port_ocean/core/integrations/mixins/sync_raw.py

@@ -233,6 +233,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         objects_diff = await self._calculate_raw(
             [(resource, results)], parse_all, send_raw_data_examples_amount
         )
+
+        ocean.metrics.inc_metric(
+            name=MetricType.OBJECT_COUNT_NAME,
+            labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM, MetricPhase.TransformResult.FAILED],
+            value=len(objects_diff[0].entity_selector_diff.failed)
+        )
+
         modified_objects = []
 
         if event.event_type == EventType.RESYNC:
@@ -395,12 +402,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             value=number_of_raw_results -number_of_transformed_entities
         )
 
-        ocean.metrics.inc_metric(
-            name=MetricType.OBJECT_COUNT_NAME,
-            labels=[ocean.metrics.current_resource_kind(), MetricPhase.TRANSFORM , MetricPhase.TransformResult.FAILED],
-            value=len(errors)
-        )
-
         return passed_entities, errors
 
     async def register_raw(

port_ocean/core/integrations/mixins/utils.py

@@ -21,7 +21,7 @@ from port_ocean.exceptions.core import (
 
 from port_ocean.utils.async_http import _http_client
 from port_ocean.clients.port.utils import _http_client as _port_http_client
-
+from port_ocean.helpers.metric.metric import MetricType, MetricPhase
 from port_ocean.context.ocean import ocean
 
 @contextmanager
@@ -61,6 +61,11 @@ async def resync_generator_wrapper(
                     yield validate_result(result)
             except OceanAbortException as error:
                 errors.append(error)
+                ocean.metrics.inc_metric(
+                    name=MetricType.OBJECT_COUNT_NAME,
+                    labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT , MetricPhase.ExtractResult.FAILED],
+                    value=1
+                )
     except StopAsyncIteration:
         if errors:
             raise ExceptionGroup(

port_ocean/helpers/metric/metric.py

@@ -37,6 +37,7 @@ class MetricPhase:
 
     class ExtractResult:
         EXTRACTED = "raw_extracted"
+        FAILED = "failed"
 
     class DeletionResult:
         DELETED = "deleted"

port_ocean/tests/core/handlers/mixins/test_sync_raw.py

@@ -98,7 +98,6 @@ async def test_sync_raw_mixin_self_dependency(
 ) -> None:
     mock_ocean.config.upsert_entities_batch_max_length = 20
     mock_ocean.config.upsert_entities_batch_max_size_in_bytes = 1024 * 1024
-    mock_ocean.config.bulk_upserts_enabled = True
 
     entities_params = [
         ("entity_1", "service", {"service": "entity_1"}, True),
@@ -219,7 +218,6 @@ async def test_sync_raw_mixin_circular_dependency(
 ) -> None:
     mock_ocean.config.upsert_entities_batch_max_length = 20
     mock_ocean.config.upsert_entities_batch_max_size_in_bytes = 1024 * 1024
-    mock_ocean.config.bulk_upserts_enabled = True
 
     entities_params = [
         ("entity_1", "service", {"service": "entity_2"}, True),
@@ -359,7 +357,6 @@ async def test_sync_raw_mixin_dependency(
 ) -> None:
     mock_ocean.config.upsert_entities_batch_max_length = 20
     mock_ocean.config.upsert_entities_batch_max_size_in_bytes = 1024 * 1024
-    mock_ocean.config.bulk_upserts_enabled = True
 
     entities_params = [
         ("entity_1", "service", {"service": "entity_3"}, True),

{port_ocean-0.24.2.dist-info → port_ocean-0.24.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.24.2
+Version: 0.24.4
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.24.2.dist-info → port_ocean-0.24.4.dist-info}/RECORD

@@ -59,7 +59,7 @@ port_ocean/clients/port/authentication.py,sha256=r7r8Ag9WuwXy-CmgeOoj-PHbmJAQxhb
 port_ocean/clients/port/client.py,sha256=dv0mxIOde6J-wFi1FXXZkoNPVHrZzY7RSMhNkDD9xgA,3566
 port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/mixins/blueprints.py,sha256=aMCG4zePsMSMjMLiGrU37h5z5_ElfMzTcTvqvOI5wXY,4683
-port_ocean/clients/port/mixins/entities.py,sha256=ZhHGasXLHGixDno2E9nMkjnE0XWlVTxlpR-0bntMX8Y,23303
+port_ocean/clients/port/mixins/entities.py,sha256=hg4xJCcYml2JLkqEwg7lJUhB5YLm_sadNfiKdf3rP6Q,22811
 port_ocean/clients/port/mixins/integrations.py,sha256=s6paomK9bYWW-Tu3y2OIaEGSxsXCHyhapVi4JIhhO64,11162
 port_ocean/clients/port/mixins/migrations.py,sha256=vdL_A_NNUogvzujyaRLIoZEu5vmKDY2BxTjoGP94YzI,1467
 port_ocean/clients/port/mixins/organization.py,sha256=A2cP5V49KnjoAXxjmnm_XGth4ftPSU0qURNfnyUyS_Y,1041
@@ -69,7 +69,7 @@ port_ocean/clients/port/utils.py,sha256=osFyAjw7Y5Qf2uVSqC7_RTCQfijiL1zS74JJM0go
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=x1gFbzujrxn7EJudRT81C6eN9WsYAb3vOHwcpcpX8Tc,6370
 port_ocean/config/dynamic.py,sha256=T0AWE41tjp9fL1sgrTRwNAGlPw6xiakFp-KXWvHtu_4,2035
-port_ocean/config/settings.py,sha256=QrxhVx1WV0GT3VUPIyRmbn5a_RJR-AWeVZWnBV63AAk,7195
+port_ocean/config/settings.py,sha256=R4Ju15XKbwQEg2W7uUCxoj4_9gUS9uYUFQnX-FUNRDI,7156
 port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
 port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -121,8 +121,8 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=8HklZmlyffYY_LeDe8xbt3Tb08rlLkqVhFF-2NQeJP4,4126
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=4zNsf0QdvsLANhoUTUnf0bNbHIyV7-MqQOYIr0ES85o,34121
-port_ocean/core/integrations/mixins/utils.py,sha256=g1XbC12dswefQ-NpcLSCqFtd_WRp2bTL98jyZ5rRbGk,3444
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=XCi6GoKgSFZjgx-pcQ7niDE0bQS-gwu4bCyqUbYqtiM,34158
+port_ocean/core/integrations/mixins/utils.py,sha256=N76dNu1Y6UEg0_pcSdF4RO6dQIZ8EBfX3xMelgWdMxM,3779
 port_ocean/core/models.py,sha256=NYsOBtAqRgmRTb2XYGDW31IxTHSXGRQLOF64apwUZ2Q,2872
 port_ocean/core/ocean_types.py,sha256=4VipWFOHEh_d9LmWewQccwx1p2dtrRYW0YURVgNsAjo,1398
 port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
@@ -139,7 +139,7 @@ port_ocean/exceptions/utils.py,sha256=gjOqpi-HpY1l4WlMFsGA9yzhxDhajhoGGdDDyGbLnq
 port_ocean/exceptions/webhook_processor.py,sha256=yQYazg53Y-ohb7HfViwq1opH_ZUuUdhHSRxcUNveFpI,114
 port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/helpers/async_client.py,sha256=SRlP6o7_FCSY3UHnRlZdezppePVxxOzZ0z861vE3K40,1783
-port_ocean/helpers/metric/metric.py,sha256=akRZmjzQdY1WMY2O3pDuU0xyW_Tn3XGttG5CkT9_Cbo,14503
+port_ocean/helpers/metric/metric.py,sha256=Aacz7bOd8ZCwEPpXAdwLbKRXf28Z4wiViG_GXiV_xWg,14529
 port_ocean/helpers/metric/utils.py,sha256=1lAgrxnZLuR_wUNDyPOPzLrm32b8cDdioob2lvnPQ1A,1619
 port_ocean/helpers/retry.py,sha256=gmS4YxM6N4fboFp7GSgtOzyBJemxs46bnrz4L4rDS6Y,16136
 port_ocean/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -166,7 +166,7 @@ port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcG
 port_ocean/tests/core/handlers/entities_state_applier/test_applier.py,sha256=eJYXc7AwrV0XRS6HpixwzghjB3pspT5Gxr9twvJE7fk,8290
 port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=8WpMn559Mf0TFWmloRpZrVgr6yWwyA0C4n2lVHCtyq4,13596
 port_ocean/tests/core/handlers/mixins/test_live_events.py,sha256=iAwVpr3n3PIkXQLw7hxd-iB_SR_vyfletVXJLOmyz28,12480
-port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256=E_r_L6Jr6DTPyUP5fn3yQ-MAC3YbpO2AHf690NyLW4g,44447
+port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256=-05ec3gRsmnMgmqzkIRjpm_yMdRZc3O3Br3RLFW2Kjw,44297
 port_ocean/tests/core/handlers/port_app_config/test_api.py,sha256=eJZ6SuFBLz71y4ca3DNqKag6d6HUjNJS0aqQPwiLMTI,1999
 port_ocean/tests/core/handlers/port_app_config/test_base.py,sha256=hSh556bJM9zuELwhwnyKSfd9z06WqWXIfe-6hCl5iKI,9799
 port_ocean/tests/core/handlers/queue/test_local_queue.py,sha256=9Ly0HzZXbs6Rbl_bstsIdInC3h2bgABU3roP9S_PnJM,2582
@@ -200,8 +200,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.24.2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.24.2.dist-info/METADATA,sha256=VvYG5ek9ZHp-Wc9vnuu8pXVOaNC2CTZ3iWy_7-vhNLQ,6764
-port_ocean-0.24.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.24.2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.24.2.dist-info/RECORD,,
+port_ocean-0.24.4.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.24.4.dist-info/METADATA,sha256=qlhSoMjy8yHoOBUz5-5MRml0HTjQDh8wKFMyatE3nXs,6764
+port_ocean-0.24.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.24.4.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.24.4.dist-info/RECORD,,