port-ocean 0.12.0__py3-none-any.whl → 0.12.2__py3-none-any.whl

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -47,37 +47,48 @@ class JQEntityProcessor(BaseEntityProcessor):
             pattern = "def env: {}; {} as $ENV | " + pattern
         return jq.compile(pattern)
 
+    @staticmethod
+    def _stop_iterator_handler(func: Any) -> Any:
+        """
+        Wrap the function to handle StopIteration exceptions.
+        Prevents StopIteration from stopping the thread and skipping further queue processing.
+        """
+
+        def inner() -> Any:
+            try:
+                return func()
+            except StopIteration:
+                return None
+
+        return inner
+
     async def _search(self, data: dict[str, Any], pattern: str) -> Any:
         try:
             loop = asyncio.get_event_loop()
             compiled_pattern = self._compile(pattern)
             func = compiled_pattern.input_value(data)
-            return await loop.run_in_executor(None, func.first)
+            return await loop.run_in_executor(
+                None, self._stop_iterator_handler(func.first)
+            )
         except Exception as exc:
             logger.debug(
-                f"Failed to search for pattern {pattern} in data {data}, {exc}"
+                f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
             )
             return None
 
     async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
         loop = asyncio.get_event_loop()
-        start_time = loop.time()
+
         compiled_pattern = self._compile(pattern)
         func = compiled_pattern.input_value(data)
-        compile_time = loop.time() - start_time
-        value = await loop.run_in_executor(None, func.first)
-        execute_time = loop.time() - start_time - compile_time
-        logger.debug(
-            f"Search for pattern {execute_time:.2f} seconds, compile time {compile_time:.2f} seconds",
-            pattern=pattern,
-            compile_time=compile_time,
-            execute_time=execute_time,
+
+        value = await loop.run_in_executor(
+            None, self._stop_iterator_handler(func.first)
         )
         if isinstance(value, bool):
             return value
-
         raise EntityProcessorException(
-            f"Expected boolean value, got {type(value)} instead"
+            f"Expected boolean value, got value:{value} of type: {type(value)} instead"
         )
 
     async def _search_as_object(

port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -195,10 +195,45 @@ class TestJQEntityProcessor:
         pattern = ".foo"
         with pytest.raises(
             EntityProcessorException,
-            match="Expected boolean value, got <class 'str'> instead",
+            match="Expected boolean value, got value:bar of type: <class 'str'> instead",
         ):
             await mocked_processor._search_as_bool(data, pattern)
 
+    @pytest.mark.parametrize(
+        "pattern, expected",
+        [
+            ('.parameters[] | select(.name == "not_exists") | .value', None),
+            (
+                '.parameters[] | select(.name == "parameter_name") | .value',
+                "parameter_value",
+            ),
+            (
+                '.parameters[] | select(.name == "another_parameter") | .value',
+                "another_value",
+            ),
+        ],
+    )
+    async def test_search_fails_on_stop_iteration(
+        self, mocked_processor: JQEntityProcessor, pattern: str, expected: Any
+    ) -> None:
+        data = {
+            "parameters": [
+                {"name": "parameter_name", "value": "parameter_value"},
+                {"name": "another_parameter", "value": "another_value"},
+                {"name": "another_parameter", "value": "another_value2"},
+            ]
+        }
+        result = await mocked_processor._search(data, pattern)
+        assert result == expected
+
+    async def test_return_a_list_of_values(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        data = {"parameters": ["parameter_value", "another_value", "another_value2"]}
+        pattern = ".parameters"
+        result = await mocked_processor._search(data, pattern)
+        assert result == ["parameter_value", "another_value", "another_value2"]
+
     @pytest.mark.timeout(3)
     async def test_search_performance_10000(
         self, mocked_processor: JQEntityProcessor

port_ocean/tests/utils/test_async_iterators.py

@@ -0,0 +1,45 @@
+from typing import Any, AsyncGenerator
+import asyncio
+from port_ocean.utils.async_iterators import semaphore_async_iterator
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_semaphore_async_iterator() -> None:
+    max_concurrency = 5
+    semaphore = asyncio.BoundedSemaphore(max_concurrency)
+
+    concurrent_tasks = 0
+    max_concurrent_tasks = 0
+    lock = asyncio.Lock()  # Protect shared variables
+
+    num_tasks = 20
+
+    async def mock_function() -> AsyncGenerator[str, None]:
+        nonlocal concurrent_tasks, max_concurrent_tasks
+
+        async with lock:
+            concurrent_tasks += 1
+            if concurrent_tasks > max_concurrent_tasks:
+                max_concurrent_tasks = concurrent_tasks
+
+        await asyncio.sleep(0.1)
+        yield "result"
+
+        async with lock:
+            concurrent_tasks -= 1
+
+    async def consume_iterator(async_iterator: Any) -> None:
+        async for _ in async_iterator:
+            pass
+
+    tasks = [
+        consume_iterator(semaphore_async_iterator(semaphore, mock_function))
+        for _ in range(num_tasks)
+    ]
+    await asyncio.gather(*tasks)
+
+    assert (
+        max_concurrent_tasks <= max_concurrency
+    ), f"Max concurrent tasks {max_concurrent_tasks} exceeded semaphore limit {max_concurrency}"
+    assert concurrent_tasks == 0, "Not all tasks have completed"

port_ocean/utils/async_iterators.py

@@ -2,6 +2,9 @@ import typing
 
 import aiostream
 
+if typing.TYPE_CHECKING:
+    from asyncio import Semaphore
+
 
 async def stream_async_iterators_tasks(
     *tasks: typing.AsyncIterable[typing.Any],
@@ -47,3 +50,60 @@ async def stream_async_iterators_tasks(
     async with combine.stream() as streamer:
         async for batch_items in streamer:
             yield batch_items
+
+
+async def semaphore_async_iterator(
+    semaphore: "Semaphore",
+    function: typing.Callable[[], typing.AsyncIterator[typing.Any]],
+) -> typing.AsyncIterator[typing.Any]:
+    """
+    Executes an asynchronous iterator function under a semaphore to limit concurrency.
+
+    This function ensures that the provided asynchronous iterator function is executed
+    while respecting the concurrency limit imposed by the semaphore. It acquires the
+    semaphore before executing the function and releases it after the function completes,
+    thus controlling the number of concurrent executions.
+
+    Parameters:
+        semaphore (asyncio.Semaphore | asyncio.BoundedSemaphore): The semaphore used to limit concurrency.
+        function (Callable[[], AsyncIterator[Any]]): A nullary asynchronous function, - apply arguments with `functools.partial` or an anonymous function (lambda)
+            that returns an asynchronous iterator. This function is executed under the semaphore.
+
+    Yields:
+        Any: The items yielded by the asynchronous iterator function.
+
+    Usage:
+        ```python
+        import asyncio
+
+        async def async_iterator_function(param1, param2):
+            # Your async code here
+            yield ...
+
+        async def async_generator_function():
+            # Your async code to retrieve items
+            param1 = "your_param1"
+            yield param1
+
+        async def main():
+            semaphore = asyncio.BoundedSemaphore(50)
+            param2 = "your_param2"
+
+            tasks = [
+                semaphore_async_iterator(
+                    semaphore,
+                    lambda: async_iterator_function(param1, param2) # functools.partial(async_iterator_function, param1, param2)
+                )
+                async for param1 in async_generator_function()
+            ]
+
+            async for batch in stream_async_iterators_tasks(*tasks):
+                # Process each batch
+                pass
+
+        asyncio.run(main())
+        ```
+    """
+    async with semaphore:
+        async for result in function():
+            yield result

{port_ocean-0.12.0.dist-info → port_ocean-0.12.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.12.0
+Version: 0.12.2
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.12.0.dist-info → port_ocean-0.12.2.dist-info}/RECORD

@@ -81,7 +81,7 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=82BvU8t5w9uhsxX8hbnwuRPuWhW3cMeuT_5sVIkip1I,1550
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=udR0w5TstTOS5xOfTjAZIEdldn4xr6Oyb3DylatYX3Q,1869
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=HQ3lnOqNITHxSn0_9TXZFaQYfMZvvPaJwnsvlNKGDKQ,8703
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=EHxU5PxvGxKJn5gKRO01bMR9PmXM6NC_NZ1L2xqlQsI,8868
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=6VbKPwFzsWG0IYsVD81hxSmfqtHUFqrfUuj1DBX5g4w,853
 port_ocean/core/handlers/port_app_config/base.py,sha256=4Nxt2g8voEIHJ4Y1Km5NJcaG2iSbCklw5P8-Kus7Y9k,3007
@@ -122,7 +122,7 @@ port_ocean/sonar-project.properties,sha256=X_wLzDOkEVmpGLRMb2fg9Rb0DxWwUFSvESId8
 port_ocean/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/tests/clients/port/mixins/test_entities.py,sha256=A9myrnkLhKSQrnOLv1Zz2wiOVSxW65Q9RIUIRbn_V7w,1586
 port_ocean/tests/conftest.py,sha256=JXASSS0IY0nnR6bxBflhzxS25kf4iNaABmThyZ0mZt8,101
-port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=fqhlLTFkP0SS12bqViYWgqayNGXWZhqpDSJObHVRnNg,9472
+port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=Yv03P-LDcJCKZ21exiTFrcT1eu0zn6Z954dilxrb52Y,10842
 port_ocean/tests/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/tests/helpers/fixtures.py,sha256=blc4ZgPEkKOmmwT6GVxceS9r1ERUwSdOIBGxWFwvRyY,1398
 port_ocean/tests/helpers/integration.py,sha256=_RxS-RHpu11lrbhUXYPZp862HLWx8AoD7iZM6iXN8rs,1104
@@ -130,9 +130,10 @@ port_ocean/tests/helpers/ocean_app.py,sha256=Dp1bwEDhWsx_G-KVxOfJX1eVIS4168ajLu3
 port_ocean/tests/helpers/port_client.py,sha256=5d6GNr8vNNSOkrz1AdOhxBUKuusr_-UPDP7AVpHasQw,599
 port_ocean/tests/helpers/smoke_test.py,sha256=_9aJJFRfuGJEg2D2YQJVJRmpreS6gEPHHQq8Q01x4aQ,2697
 port_ocean/tests/test_smoke.py,sha256=uix2uIg_yOm8BHDgHw2hTFPy1fiIyxBGW3ENU_KoFlo,2557
+port_ocean/tests/utils/test_async_iterators.py,sha256=3PLk1emEXekb8LcC5GgVh3OicaX15i5WyaJT_eFnu_4,1336
 port_ocean/utils/__init__.py,sha256=KMGnCPXZJbNwtgxtyMycapkDz8tpSyw23MSYT3iVeHs,91
 port_ocean/utils/async_http.py,sha256=arnH458TExn2Dju_Sy6pHas_vF5RMWnOp-jBz5WAAcE,1226
-port_ocean/utils/async_iterators.py,sha256=iw3cUHxfQm3zUSPdw2FmSXDU8E1Ppnys4TGhswNuQ8s,1569
+port_ocean/utils/async_iterators.py,sha256=CPXskYWkhkZtAG-ducEwM8537t3z5usPEqXR9vcivzw,3715
 port_ocean/utils/cache.py,sha256=3KItZDE2yVrbVDr-hoM8lNna8s2dlpxhP4ICdLjH4LQ,2231
 port_ocean/utils/misc.py,sha256=0q2cJ5psqxn_5u_56pT7vOVQ3shDM02iC1lzyWQ_zl0,2098
 port_ocean/utils/queue_utils.py,sha256=KWWl8YVnG-glcfIHhM6nefY-2sou_C6DVP1VynQwzB4,2762
@@ -140,8 +141,8 @@ port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,32
 port_ocean/utils/signal.py,sha256=K-6kKFQTltcmKDhtyZAcn0IMa3sUpOHGOAUdWKgx0_E,1369
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.12.0.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.12.0.dist-info/METADATA,sha256=PrJZo5rBOy6h2_slD1tjSHcTdHCc-PgyIsk9DNqk0pY,6614
-port_ocean-0.12.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-port_ocean-0.12.0.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.12.0.dist-info/RECORD,,
+port_ocean-0.12.2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.12.2.dist-info/METADATA,sha256=CB3srEnScX99emeWn9qiASGQ6hRn2KO2DiCEEi1pMXs,6614
+port_ocean-0.12.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+port_ocean-0.12.2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.12.2.dist-info/RECORD,,