port-ocean 0.27.1__py3-none-any.whl → 0.27.2__py3-none-any.whl

port_ocean/config/settings.py

@@ -111,6 +111,8 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     upsert_entities_batch_max_length: int = 20
     upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
     lakehouse_enabled: bool = False
+    yield_items_to_parse: bool = False
+    yield_items_to_parse_batch_size: int = 10
 
     @validator("process_execution_mode")
     def validate_process_execution_mode(

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -246,15 +246,16 @@ class JQEntityProcessor(BaseEntityProcessor):
         parse_all: bool = False,
     ) -> tuple[list[MappedEntity], list[Exception]]:
         raw_data = [data.copy()]
-        if items_to_parse:
-            items = await self._search(data, items_to_parse)
-            if not isinstance(items, list):
-                logger.warning(
-                    f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                    f" Skipping..."
-                )
-                return [], []
-            raw_data = [{"item": item, **data} for item in items]
+        if not ocean.config.yield_items_to_parse:
+            if items_to_parse:
+                items = await self._search(data, items_to_parse)
+                if not isinstance(items, list):
+                    logger.warning(
+                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                        f" Skipping..."
+                    )
+                    return [], []
+                raw_data = [{"item": item, **data} for item in items]
 
         entities, errors = await gather_and_split_errors_from_results(
             [

port_ocean/core/integrations/mixins/sync_raw.py

@@ -116,7 +116,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind))
+                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"

port_ocean/core/integrations/mixins/utils.py

@@ -1,11 +1,13 @@
 from contextlib import contextmanager
-from typing import Awaitable, Generator, Callable
+from typing import Awaitable, Generator, Callable, cast
 
 from loguru import logger
 
 import asyncio
 import multiprocessing
 
+from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -49,7 +51,7 @@ async def resync_function_wrapper(
 
 
 async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
 ) -> ASYNC_GENERATOR_RESYNC_TYPE:
     generator = fn(kind)
     errors = []
@@ -58,7 +60,28 @@ async def resync_generator_wrapper(
             try:
                 with resync_error_handling():
                     result = await anext(generator)
-                    yield validate_result(result)
+                    if not ocean.config.yield_items_to_parse:
+                        yield validate_result(result)
+                    else:
+                        batch_size = ocean.config.yield_items_to_parse_batch_size
+                        if items_to_parse:
+                            for data in result:
+                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
+                                if not isinstance(items, list):
+                                    logger.warning(
+                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                                        f" Skipping..."
+                                    )
+                                    yield []
+                                raw_data = [{"item": item, **data} for item in items]
+                                while True:
+                                    raw_data_batch = raw_data[:batch_size]
+                                    yield raw_data_batch
+                                    raw_data = raw_data[batch_size:]
+                                    if len(raw_data) == 0:
+                                        break
+                        else:
+                            yield validate_result(result)
             except OceanAbortException as error:
                 errors.append(error)
                 ocean.metrics.inc_metric(

{port_ocean-0.27.1.dist-info → port_ocean-0.27.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.27.1
+Version: 0.27.2
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.27.1.dist-info → port_ocean-0.27.2.dist-info}/RECORD

@@ -70,7 +70,7 @@ port_ocean/clients/port/utils.py,sha256=osFyAjw7Y5Qf2uVSqC7_RTCQfijiL1zS74JJM0go
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=x1gFbzujrxn7EJudRT81C6eN9WsYAb3vOHwcpcpX8Tc,6370
 port_ocean/config/dynamic.py,sha256=Lrk4JRGtR-0YKQ9DDGexX5NGFE7EJ6VoHya19YYhssM,2687
-port_ocean/config/settings.py,sha256=keRT2FJyzQ2G0LNlNP4mqYSwwTjbybL7Ire0oggwScw,7226
+port_ocean/config/settings.py,sha256=-jgVUBiFbeVjd9GjkMH6cRLLQHhtvI9x1WNdVE5s_p0,7311
 port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
 port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -101,7 +101,7 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=PsnpNRqjHth9xwOvDRe7gKu8cjnVV0XGmTIHGvOelX0,1867
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=GxK5BM0yzAoeYTK2-ALKh5qlAe9pebcN7vZfQCZjksM,12787
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=t_Z6etgxG6IQLNRLxG06SEj5IyAVRujuf0ox0DXd4A8,12873
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=r_Th66NEw38IpRdnXZcRvI8ACfvxW_A6V62WLwjWXlQ,1044
 port_ocean/core/handlers/port_app_config/base.py,sha256=Sup4-X_a7JGa27rMy_OgqGIjFHMlKBpKevicaK3AeHU,2919
@@ -123,8 +123,8 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=6AHuroSoqIKvdx2BAKtILxfQFeYB_wikYaM4JaTeoGw,39315
-port_ocean/core/integrations/mixins/utils.py,sha256=N76dNu1Y6UEg0_pcSdF4RO6dQIZ8EBfX3xMelgWdMxM,3779
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=OptSievJ1c9vw-0PKmYuT2GgbCH6HC04yEyi0MUQF9k,39351
+port_ocean/core/integrations/mixins/utils.py,sha256=r7VKatZDip5D0hu-chCK3IeuM1gItleOEJboNN0Zvdk,5309
 port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
 port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
 port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
@@ -205,8 +205,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.27.1.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.27.1.dist-info/METADATA,sha256=lMu1khMLwRsJuGctmRMHCZMr6fGDaXpSK53uQ0Iby1Y,6887
-port_ocean-0.27.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.27.1.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.27.1.dist-info/RECORD,,
+port_ocean-0.27.2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.27.2.dist-info/METADATA,sha256=i1U80P8ZYN4o6kNPfWsYpcCnPlQsBSoh0llLjg4Dk7A,6887
+port_ocean-0.27.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.27.2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.27.2.dist-info/RECORD,,