port-ocean 0.5.17__py3-none-any.whl → 0.5.19__py3-none-any.whl

port_ocean/config/settings.py

@@ -36,6 +36,7 @@ class PortSettings(BaseOceanModel, extra=Extra.allow):
     client_id: str = Field(..., sensitive=True)
     client_secret: str = Field(..., sensitive=True)
     base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io")
+    port_app_config_cache_ttl: int = 60
 
 
 class IntegrationSettings(BaseOceanModel, extra=Extra.allow):

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -14,6 +14,7 @@ from port_ocean.core.ocean_types import (
     EntitySelectorDiff,
 )
 from port_ocean.exceptions.core import EntityProcessorException
+from port_ocean.utils.queue_utils import process_in_queue
 
 
 class JQEntityProcessor(BaseEntityProcessor):
@@ -128,23 +129,19 @@ class JQEntityProcessor(BaseEntityProcessor):
         raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
             exclude_unset=True
         )
-        calculate_entities_tasks = [
-            asyncio.create_task(
-                self._calculate_entity(
-                    data,
-                    raw_entity_mappings,
-                    mapping.port.items_to_parse,
-                    mapping.selector.query,
-                    parse_all,
-                )
-            )
-            for data in raw_results
-        ]
-        calculate_entities_results = await asyncio.gather(*calculate_entities_tasks)
+
+        calculated_entities_results = await process_in_queue(
+            raw_results,
+            self._calculate_entity,
+            raw_entity_mappings,
+            mapping.port.items_to_parse,
+            mapping.selector.query,
+            parse_all,
+        )
 
         passed_entities = []
         failed_entities = []
-        for entities_results in calculate_entities_results:
+        for entities_results in calculated_entities_results:
             for entity, did_entity_pass_selector in entities_results:
                 if entity.get("identifier") and entity.get("blueprint"):
                     parsed_entity = Entity.parse_obj(entity)

port_ocean/core/handlers/port_app_config/base.py

@@ -5,8 +5,36 @@ from loguru import logger
 from pydantic import ValidationError
 
 from port_ocean.context.event import event
+from port_ocean.context.ocean import PortOceanContext
 from port_ocean.core.handlers.base import BaseHandler
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
+from port_ocean.utils.misc import get_time
+
+
+class PortAppConfigCache:
+    _port_app_config: PortAppConfig | None
+    _retrieval_time: float
+
+    def __init__(self, cache_ttl: int):
+        self._cache_ttl = cache_ttl
+
+    @property
+    def port_app_config(self) -> PortAppConfig:
+        if self._port_app_config is None:
+            raise ValueError("Port app config is not set")
+        return self._port_app_config
+
+    @port_app_config.setter
+    def port_app_config(self, value: PortAppConfig) -> None:
+        self._retrieval_time = get_time()
+        self._port_app_config = value
+
+    @property
+    def is_cache_invalid(self) -> bool:
+        return (
+            not self._port_app_config
+            or self._retrieval_time + self._cache_ttl < get_time()
+        )
 
 
 class BasePortAppConfig(BaseHandler):
@@ -21,24 +49,34 @@ class BasePortAppConfig(BaseHandler):
 
     CONFIG_CLASS: Type[PortAppConfig] = PortAppConfig
 
+    def __init__(self, context: PortOceanContext):
+        super().__init__(context)
+        self._app_config_cache = PortAppConfigCache(
+            self.context.config.port.port_app_config_cache_ttl
+        )
+
     @abstractmethod
     async def _get_port_app_config(self) -> dict[str, Any]:
         pass
 
-    async def get_port_app_config(self) -> PortAppConfig:
-        """Retrieve and parse the port application configuration.
+    async def get_port_app_config(self, use_cache: bool = True) -> PortAppConfig:
+        """
+        Retrieve and parse the port application configuration.
 
-        Returns:
-            PortAppConfig: The parsed port application configuration.
+        :param use_cache: Determines whether to use the cached port-app-config if it exists, or to fetch it regardless
+        :return: The parsed port application configuration.
         """
-        raw_config = await self._get_port_app_config()
-        try:
-            config = self.CONFIG_CLASS.parse_obj(raw_config)
-        except ValidationError:
-            logger.error(
-                "Invalid port app config found. Please check that the integration has been configured correctly."
-            )
-            raise
-
-        event.port_app_config = config
-        return config
+        if not use_cache or self._app_config_cache.is_cache_invalid:
+            raw_config = await self._get_port_app_config()
+            try:
+                self._app_config_cache.port_app_config = self.CONFIG_CLASS.parse_obj(
+                    raw_config
+                )
+            except ValidationError:
+                logger.error(
+                    "Invalid port app config found. Please check that the integration has been configured correctly."
+                )
+                raise
+
+        event.port_app_config = self._app_config_cache.port_app_config
+        return self._app_config_cache.port_app_config

port_ocean/core/integrations/mixins/sync_raw.py

@@ -362,7 +362,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             EventType.RESYNC,
             trigger_type=trigger_type,
         ):
-            app_config = await self.port_app_config_handler.get_port_app_config()
+            # If a resync is triggered due to a mappings change, we want to make sure that we have the updated version
+            # rather than the old cache
+            app_config = await self.port_app_config_handler.get_port_app_config(
+                use_cache=False
+            )
 
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 

port_ocean/core/utils.py

@@ -28,38 +28,34 @@ def is_same_entity(first_entity: Entity, second_entity: Entity) -> bool:
     )
 
 
-def get_unique(array: list[Entity]) -> list[Entity]:
-    result: list[Entity] = []
-    for item in array:
-        if all(not is_same_entity(item, seen_item) for seen_item in result):
-            result.append(item)
-    return result
-
-
 def get_port_diff(
     before: Iterable[Entity],
     after: Iterable[Entity],
 ) -> EntityPortDiff:
-    return EntityPortDiff(
-        deleted=get_unique(
-            [
-                item
-                for item in before
-                if not any(is_same_entity(item, item_after) for item_after in after)
-            ],
-        ),
-        created=get_unique(
-            [
-                item
-                for item in after
-                if not any(is_same_entity(item, item_before) for item_before in before)
-            ],
-        ),
-        modified=get_unique(
-            [
-                item
-                for item in after
-                if any(is_same_entity(item, item_before) for item_before in before)
-            ],
-        ),
-    )
+    before_dict = {}
+    after_dict = {}
+    created = []
+    modified = []
+    deleted = []
+
+    # Create dictionaries for before and after lists
+    for entity in before:
+        key = (entity.identifier, entity.blueprint)
+        before_dict[key] = entity
+
+    for entity in after:
+        key = (entity.identifier, entity.blueprint)
+        after_dict[key] = entity
+
+    # Find created, modified, and deleted objects
+    for key, obj in after_dict.items():
+        if key not in before_dict:
+            created.append(obj)
+        else:
+            modified.append(obj)
+
+    for key, obj in before_dict.items():
+        if key not in after_dict:
+            deleted.append(obj)
+
+    return EntityPortDiff(created=created, modified=modified, deleted=deleted)

port_ocean/utils/queue_utils.py

@@ -0,0 +1,81 @@
+import asyncio
+from asyncio import Queue, Task
+from typing import Any, TypeVar, Callable, Coroutine
+
+from loguru import logger
+
+T = TypeVar("T")
+
+
+async def _start_processor_worker(
+    queue: Queue[Any | None],
+    func: Callable[..., Coroutine[Any, Any, T]],
+    results: list[T],
+) -> None:
+    while True:
+        raw_params = await queue.get()
+        try:
+            if raw_params is None:
+                return
+            logger.debug(f"Processing {raw_params[0]}")
+            results.append(await func(*raw_params))
+        finally:
+            queue.task_done()
+
+
+async def process_in_queue(
+    objects_to_process: list[Any],
+    func: Callable[..., Coroutine[Any, Any, T]],
+    *args: Any,
+    concurrency: int = 50,
+) -> list[T]:
+    """
+    This function executes multiple asynchronous tasks in a bounded way
+    (e.g. having 200 tasks to execute, while running only 20 concurrently),
+    to prevent overload and memory issues when dealing with large sets of data and tasks.
+    read more -> https://stackoverflow.com/questions/38831322/making-1-milion-requests-with-aiohttp-asyncio-literally
+
+    Usage:
+    ```python
+    async def incrementBy(num: int, increment_by: int) -> int:
+        await asyncio.sleep(3)
+        return num + increment_by
+
+    async def main():
+        raw_objects = [1, 2, 3, 4, 5]
+        processed_objects = await process_in_queue(
+            raw_objects,
+            incrementBy,
+            5
+        )
+    ```
+
+    :param objects_to_process: A list of the raw objects to process
+    :param func: An async function that turns raw object into result object
+    :param args: Static arguments to pass to the func when called
+    :param concurrency: An integer specifying the concurrent workers count
+    :return: A list of result objects
+    """
+    queue: Queue[Any | None] = Queue(maxsize=concurrency * 2)
+    tasks: list[Task[Any]] = []
+    processing_results: list[T] = []
+
+    for i in range(concurrency):
+        tasks.append(
+            asyncio.create_task(
+                _start_processor_worker(queue, func, processing_results)
+            )
+        )
+
+    for i in range(len(objects_to_process)):
+        await queue.put((objects_to_process[i], *args))
+
+    for i in range(concurrency):
+        # We put None value into the queue, so the workers will know that we
+        # are done sending more input and they can terminate
+        await queue.put(None)
+
+    await queue.join()
+    await asyncio.gather(*tasks)
+
+    return processing_results

{port_ocean-0.5.17.dist-info → port_ocean-0.5.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.5.17
+Version: 0.5.19
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.5.17.dist-info → port_ocean-0.5.19.dist-info}/RECORD

@@ -49,7 +49,7 @@ port_ocean/clients/port/utils.py,sha256=O9mBu6zp4TfpS4SQ3qCPpn9ZVyYF8GKnji4UnYhM
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=8JoyjTPJS_BeYMnGnKN_q4SDaI-ydctuKl9ccK-Uq78,6474
 port_ocean/config/dynamic.py,sha256=qOFkRoJsn_BW7581omi_AoMxoHqasf_foxDQ_G11_SI,2030
-port_ocean/config/settings.py,sha256=5bQmOAIZ2IMDzUEsiVkfZokv9LxHLPOUVKA0TPoJh68,1769
+port_ocean/config/settings.py,sha256=7eJSuOQU69NWrxUfy6UnSdL9aZQ_nu9VOp0-XHy9-Ds,1809
 port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
 port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -78,10 +78,10 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=82BvU8t5w9uhsxX8hbnwuRPuWhW3cMeuT_5sVIkip1I,1550
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=4JVCAAohEKtl8FdlnuyIxJ1afSXk3o2-e_m4LSy7vmw,1952
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=R5PwTcsJYRVU-lwoYZa1Af8kserp2BXIpyzVtRdEqA8,5617
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=FDpMeBntu5x-p2DGuk8BpQVe1zy7afJ93nWfrLnZPRc,5479
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=6VbKPwFzsWG0IYsVD81hxSmfqtHUFqrfUuj1DBX5g4w,853
-port_ocean/core/handlers/port_app_config/base.py,sha256=nnMZ4jH6a-4Of9Cn-apMsU0CgNLD9avd5q0gRmc7nZ8,1495
+port_ocean/core/handlers/port_app_config/base.py,sha256=oufdNLzUmEgJY5PgIz75zDlowWrjA-y8WR4UnM58E58,2897
 port_ocean/core/handlers/port_app_config/models.py,sha256=WtF2uW4KPUPfDpy6E2tl9qXh5GUcDucVvKt0DCTYv6w,1985
 port_ocean/core/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/core/integrations/base.py,sha256=3jU0skK_gMLB8a_fN8whsRKva-Dz058TFoI0vXTbryU,3193
@@ -89,11 +89,11 @@ port_ocean/core/integrations/mixins/__init__.py,sha256=FA1FEKMM6P-L2_m7Q4L20mFa4
 port_ocean/core/integrations/mixins/events.py,sha256=Ddfx2L4FpghV38waF8OfVeOV0bHBxNIgjU-q5ffillI,2341
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/sync.py,sha256=TKqRytxXONVhuCo3CB3rDvWNbITnZz33TYTDs3SWWVk,3880
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=kWZ44L2M2G6B0n3oOqF1Ko9Rct06CXnTnWY25Q2eh3c,15049
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=iWI8fXd36TwUNFDAgEJj4SGvWwDOQsez1sQpc2EriKQ,15253
 port_ocean/core/integrations/mixins/utils.py,sha256=7y1rGETZIjOQadyIjFJXIHKkQFKx_SwiP-TrAIsyyLY,2303
 port_ocean/core/models.py,sha256=bDO_I4Yd33TEZIh2QSV8UwXQIuwE7IgrINkYDHI0dkc,714
 port_ocean/core/ocean_types.py,sha256=ltnn22eRuDMFW02kIgmIAu6S06-i9jJV2NJ-MZcwwj0,879
-port_ocean/core/utils.py,sha256=B040Wokk28g9tQj_06qk_uvm85RIXc6XGXysZV6gtQw,1957
+port_ocean/core/utils.py,sha256=6rYtsb1bjW8owxWngGiV3awMLZkP3tXZdxXClmRD1SU,1824
 port_ocean/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/exceptions/api.py,sha256=TLmTMqn4uHGaHgZK8PMIJ0TVJlPB4iP7xl9rx7GtCyY,426
 port_ocean/exceptions/base.py,sha256=uY4DX7fIITDFfemCJDWpaZi3bD51lcANc5swpoNvMJA,46
@@ -119,11 +119,12 @@ port_ocean/utils/async_http.py,sha256=arnH458TExn2Dju_Sy6pHas_vF5RMWnOp-jBz5WAAc
 port_ocean/utils/async_iterators.py,sha256=buFBiPdsqkNMCk91h6ZG8hJa181j7RjgHajbfgeB8A8,1608
 port_ocean/utils/cache.py,sha256=3KItZDE2yVrbVDr-hoM8lNna8s2dlpxhP4ICdLjH4LQ,2231
 port_ocean/utils/misc.py,sha256=2XmO8W0SgPjV0rd9HZvrHhoMlHprIwmMFsINxlAmgyw,1723
+port_ocean/utils/queue_utils.py,sha256=Pzb6e8PcjylZpXcb9EEIK-QcTty_E2k1egMiJF5J_8Q,2500
 port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,3231
 port_ocean/utils/signal.py,sha256=Fab0049Cjs69TPTQgvEvilaVZKACQr6tGkRdySjNCi8,1515
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.5.17.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.5.17.dist-info/METADATA,sha256=uaVWJBt0vRCLiqO22dU72mqY-rnoBZgXRKJvmCCKFzE,6554
-port_ocean-0.5.17.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-port_ocean-0.5.17.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.5.17.dist-info/RECORD,,
+port_ocean-0.5.19.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.5.19.dist-info/METADATA,sha256=m3voj8mZP19CPHPXnBDLx58G07P5kfxDuCWAjvu2wTE,6554
+port_ocean-0.5.19.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+port_ocean-0.5.19.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.5.19.dist-info/RECORD,,