port-ocean 0.27.1__py3-none-any.whl → 0.27.3__py3-none-any.whl

port_ocean/config/settings.py

@@ -111,6 +111,8 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     upsert_entities_batch_max_length: int = 20
     upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
     lakehouse_enabled: bool = False
+    yield_items_to_parse: bool = False
+    yield_items_to_parse_batch_size: int = 10
 
     @validator("process_execution_mode")
     def validate_process_execution_mode(

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -242,19 +242,21 @@ class JQEntityProcessor(BaseEntityProcessor):
         data: dict[str, Any],
         raw_entity_mappings: dict[str, Any],
         items_to_parse: str | None,
+        items_to_parse_name: str,
         selector_query: str,
         parse_all: bool = False,
     ) -> tuple[list[MappedEntity], list[Exception]]:
         raw_data = [data.copy()]
-        if items_to_parse:
-            items = await self._search(data, items_to_parse)
-            if not isinstance(items, list):
-                logger.warning(
-                    f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                    f" Skipping..."
-                )
-                return [], []
-            raw_data = [{"item": item, **data} for item in items]
+        if not ocean.config.yield_items_to_parse:
+            if items_to_parse:
+                items = await self._search(data, items_to_parse)
+                if not isinstance(items, list):
+                    logger.warning(
+                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                        f" Skipping..."
+                    )
+                    return [], []
+                raw_data = [{items_to_parse_name: item, **data} for item in items]
 
         entities, errors = await gather_and_split_errors_from_results(
             [
@@ -303,6 +305,7 @@ class JQEntityProcessor(BaseEntityProcessor):
                 self._calculate_entity,
                 raw_entity_mappings,
                 mapping.port.items_to_parse,
+                mapping.port.items_to_parse_name,
                 mapping.selector.query,
                 parse_all,
             )

port_ocean/core/handlers/port_app_config/models.py

@@ -39,6 +39,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
+    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

port_ocean/core/integrations/mixins/sync_raw.py

@@ -116,7 +116,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind))
+                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"

port_ocean/core/integrations/mixins/utils.py

@@ -1,11 +1,13 @@
 from contextlib import contextmanager
-from typing import Awaitable, Generator, Callable
+from typing import Awaitable, Generator, Callable, cast
 
 from loguru import logger
 
 import asyncio
 import multiprocessing
 
+from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -49,7 +51,7 @@ async def resync_function_wrapper(
 
 
 async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
 ) -> ASYNC_GENERATOR_RESYNC_TYPE:
     generator = fn(kind)
     errors = []
@@ -58,7 +60,28 @@ async def resync_generator_wrapper(
             try:
                 with resync_error_handling():
                     result = await anext(generator)
-                    yield validate_result(result)
+                    if not ocean.config.yield_items_to_parse:
+                        yield validate_result(result)
+                    else:
+                        batch_size = ocean.config.yield_items_to_parse_batch_size
+                        if items_to_parse:
+                            for data in result:
+                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
+                                if not isinstance(items, list):
+                                    logger.warning(
+                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                                        f" Skipping..."
+                                    )
+                                    yield []
+                                raw_data = [{"item": item, **data} for item in items]
+                                while True:
+                                    raw_data_batch = raw_data[:batch_size]
+                                    yield raw_data_batch
+                                    raw_data = raw_data[batch_size:]
+                                    if len(raw_data) == 0:
+                                        break
+                        else:
+                            yield validate_result(result)
             except OceanAbortException as error:
                 errors.append(error)
                 ocean.metrics.inc_metric(

port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -60,7 +60,7 @@ class TestJQEntityProcessor:
         raw_entity_mappings = {"foo": ".foo"}
         selector_query = '.foo == "bar"'
         result, errors = await mocked_processor._calculate_entity(
-            data, raw_entity_mappings, None, selector_query
+            data, raw_entity_mappings, None, "item", selector_query
         )
         assert len(result) == 1
         assert result[0].entity == {"foo": "bar"}

{port_ocean-0.27.1.dist-info → port_ocean-0.27.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.27.1
+Version: 0.27.3
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.27.1.dist-info → port_ocean-0.27.3.dist-info}/RECORD

@@ -70,7 +70,7 @@ port_ocean/clients/port/utils.py,sha256=osFyAjw7Y5Qf2uVSqC7_RTCQfijiL1zS74JJM0go
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=x1gFbzujrxn7EJudRT81C6eN9WsYAb3vOHwcpcpX8Tc,6370
 port_ocean/config/dynamic.py,sha256=Lrk4JRGtR-0YKQ9DDGexX5NGFE7EJ6VoHya19YYhssM,2687
-port_ocean/config/settings.py,sha256=keRT2FJyzQ2G0LNlNP4mqYSwwTjbybL7Ire0oggwScw,7226
+port_ocean/config/settings.py,sha256=-jgVUBiFbeVjd9GjkMH6cRLLQHhtvI9x1WNdVE5s_p0,7311
 port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
 port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -101,11 +101,11 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=PsnpNRqjHth9xwOvDRe7gKu8cjnVV0XGmTIHGvOelX0,1867
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=GxK5BM0yzAoeYTK2-ALKh5qlAe9pebcN7vZfQCZjksM,12787
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=qvPMbIH1XRvaZ-TvW7lw9k4W27ZPCHcXGSdqnZ0wblw,12970
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=r_Th66NEw38IpRdnXZcRvI8ACfvxW_A6V62WLwjWXlQ,1044
 port_ocean/core/handlers/port_app_config/base.py,sha256=Sup4-X_a7JGa27rMy_OgqGIjFHMlKBpKevicaK3AeHU,2919
-port_ocean/core/handlers/port_app_config/models.py,sha256=pO7oI7GIYZ9c2ZxLu8EQ97U2IPqzsbJf3gRQxlizEjE,2933
+port_ocean/core/handlers/port_app_config/models.py,sha256=M9NJRRacvpZQYCFgHGlRwrdgumbIIJ82WpCyvtOUf5w,3019
 port_ocean/core/handlers/queue/__init__.py,sha256=yzgicE_jAR1wtljFKxgyG6j-HbLcG_Zze5qw1kkALUI,171
 port_ocean/core/handlers/queue/abstract_queue.py,sha256=SaivrYbqg8qsX6wtQlJZyxgcbdMD5B9NZG3byN9AvrI,782
 port_ocean/core/handlers/queue/group_queue.py,sha256=JvvJOwz9z_aI4CjPr7yQX-0rOgqLI5wMdxWk2x5x-34,4989
@@ -123,8 +123,8 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=6AHuroSoqIKvdx2BAKtILxfQFeYB_wikYaM4JaTeoGw,39315
-port_ocean/core/integrations/mixins/utils.py,sha256=N76dNu1Y6UEg0_pcSdF4RO6dQIZ8EBfX3xMelgWdMxM,3779
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=OptSievJ1c9vw-0PKmYuT2GgbCH6HC04yEyi0MUQF9k,39351
+port_ocean/core/integrations/mixins/utils.py,sha256=r7VKatZDip5D0hu-chCK3IeuM1gItleOEJboNN0Zvdk,5309
 port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
 port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
 port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
@@ -168,7 +168,7 @@ port_ocean/tests/conftest.py,sha256=JXASSS0IY0nnR6bxBflhzxS25kf4iNaABmThyZ0mZt8,
 port_ocean/tests/core/conftest.py,sha256=5shShx81LRuQTBwS2sDIjNJO2LSD6cUNz46SgNYzjGY,7686
 port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcGsYZ2SCe_PYX5vLYM,5560
 port_ocean/tests/core/handlers/entities_state_applier/test_applier.py,sha256=7XWgwUB9uVYRov4VbIz1A-7n2YLbHTTYT-4rKJxjB0A,10711
-port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=AyzLclJFKz8YzSi2hiFYVdXRzCFmqQwTxnwNFTo9roU,14091
+port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=TjSj8ssIqH23VJlO5PGovbudCqDbuE2-54iNQsD9K-I,14099
 port_ocean/tests/core/handlers/mixins/test_live_events.py,sha256=6yUsYooBYchiZP_eYa8PN1IhiztJShBdPguoseyNMzY,12482
 port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256=-Jd2rUG63fZM8LuyKtCp1tt4WEqO2m5woESjs1c91sU,44428
 port_ocean/tests/core/handlers/port_app_config/test_api.py,sha256=eJZ6SuFBLz71y4ca3DNqKag6d6HUjNJS0aqQPwiLMTI,1999
@@ -205,8 +205,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.27.1.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.27.1.dist-info/METADATA,sha256=lMu1khMLwRsJuGctmRMHCZMr6fGDaXpSK53uQ0Iby1Y,6887
-port_ocean-0.27.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.27.1.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.27.1.dist-info/RECORD,,
+port_ocean-0.27.3.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.27.3.dist-info/METADATA,sha256=8q1fasdb2UPwIRETpBdDwk5m9IEDUOrBEBl5fSruMZ8,6887
+port_ocean-0.27.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.27.3.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.27.3.dist-info/RECORD,,