port-ocean 0.12.2__py3-none-any.whl → 0.12.2.dev2__py3-none-any.whl

port_ocean/core/event_listener/base.py

@@ -1,12 +1,14 @@
+import asyncio
 from abc import abstractmethod
+from concurrent.futures import ProcessPoolExecutor
 from typing import TypedDict, Callable, Any, Awaitable
 
 from pydantic import Extra
 
 from port_ocean.config.base import BaseOceanModel
-from port_ocean.utils.signal import signal_handler
 from port_ocean.context.ocean import ocean
 from port_ocean.utils.misc import IntegrationStateStatus
+from port_ocean.utils.signal import signal_handler
 
 
 class EventListenerEvents(TypedDict):
@@ -67,7 +69,14 @@ class BaseEventListener:
         """
         await self._before_resync()
         try:
-            await self.events["on_resync"](resync_args)
+            loop = asyncio.get_event_loop()
+            with ProcessPoolExecutor() as executor:
+                e = executor.submit(
+                    lambda: asyncio.run_coroutine_threadsafe(
+                        self.events["on_resync"](resync_args), loop
+                    )
+                )
+                signal_handler.register(e.cancel)
             await self._after_resync()
         except Exception as e:
             await self._on_resync_failure(e)

port_ocean/log/handlers.py

@@ -1,8 +1,8 @@
 import asyncio
 import logging
 import sys
-import threading
 import time
+from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from logging.handlers import MemoryHandler
 from typing import Any
@@ -65,18 +65,19 @@ class HTTPMemoryHandler(MemoryHandler):
         if self.ocean is None or not self.buffer:
             return
 
-        def _wrap_event_loop(_ocean: Ocean, logs_to_send: list[dict[str, Any]]) -> None:
-            loop = asyncio.new_event_loop()
-            loop.run_until_complete(self.send_logs(_ocean, logs_to_send))
-            loop.close()
-
         self.acquire()
         logs = list(self._serialized_buffer)
         if logs:
             self.buffer.clear()
             self._serialized_buffer.clear()
             self.last_flush_time = time.time()
-            threading.Thread(target=_wrap_event_loop, args=(self.ocean, logs)).start()
+            loop = asyncio.new_event_loop()
+            with ProcessPoolExecutor() as executor:
+                executor.submit(
+                    lambda: asyncio.run_coroutine_threadsafe(
+                        self.send_logs(self.ocean, logs), loop
+                    )
+                )
         self.release()
 
     async def send_logs(

port_ocean/ocean.py

@@ -1,6 +1,6 @@
 import asyncio
 import sys
-import threading
+from concurrent.futures import ProcessPoolExecutor
 from contextlib import asynccontextmanager
 from typing import Callable, Any, Dict, AsyncIterator, Type
 
@@ -9,8 +9,6 @@ from loguru import logger
 from pydantic import BaseModel
 from starlette.types import Scope, Receive, Send
 
-from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
-from port_ocean.core.models import Runtime
 from port_ocean.clients.port.client import PortClient
 from port_ocean.config.settings import (
     IntegrationConfiguration,
@@ -20,13 +18,15 @@ from port_ocean.context.ocean import (
     ocean,
     initialize_port_ocean_context,
 )
+from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
 from port_ocean.core.integrations.base import BaseIntegration
+from port_ocean.core.models import Runtime
 from port_ocean.log.sensetive import sensitive_log_filter
 from port_ocean.middlewares import request_handler
+from port_ocean.utils.misc import IntegrationStateStatus
 from port_ocean.utils.repeat import repeat_every
 from port_ocean.utils.signal import signal_handler
 from port_ocean.version import __integration_version__
-from port_ocean.utils.misc import IntegrationStateStatus
 
 
 class Ocean:
@@ -92,6 +92,13 @@ class Ocean:
                 )
                 raise e
 
+        def pool_executor_wrapper(event_loop) -> None:
+            with ProcessPoolExecutor() as executor:
+                e = executor.submit(lambda: asyncio.run_coroutine_threadsafe(
+                    execute_resync_all(), event_loop
+                ))
+                signal_handler.register(e.cancel)
+
         interval = self.config.scheduled_resync_interval
         loop = asyncio.get_event_loop()
         if interval is not None:
@@ -103,13 +110,7 @@ class Ocean:
                 seconds=interval * 60,
                 # Not running the resync immediately because the event listener should run resync on startup
                 wait_first=True,
-            )(
-                lambda: threading.Thread(
-                    target=lambda: asyncio.run_coroutine_threadsafe(
-                        execute_resync_all(), loop
-                    )
-                ).start()
-            )
+            )(lambda _: pool_executor_wrapper(loop))
             await repeated_function()
 
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:

port_ocean/utils/async_iterators.py

@@ -2,9 +2,6 @@ import typing
 
 import aiostream
 
-if typing.TYPE_CHECKING:
-    from asyncio import Semaphore
-
 
 async def stream_async_iterators_tasks(
     *tasks: typing.AsyncIterable[typing.Any],
@@ -50,60 +47,3 @@ async def stream_async_iterators_tasks(
     async with combine.stream() as streamer:
         async for batch_items in streamer:
             yield batch_items
-
-
-async def semaphore_async_iterator(
-    semaphore: "Semaphore",
-    function: typing.Callable[[], typing.AsyncIterator[typing.Any]],
-) -> typing.AsyncIterator[typing.Any]:
-    """
-    Executes an asynchronous iterator function under a semaphore to limit concurrency.
-
-    This function ensures that the provided asynchronous iterator function is executed
-    while respecting the concurrency limit imposed by the semaphore. It acquires the
-    semaphore before executing the function and releases it after the function completes,
-    thus controlling the number of concurrent executions.
-
-    Parameters:
-        semaphore (asyncio.Semaphore | asyncio.BoundedSemaphore): The semaphore used to limit concurrency.
-        function (Callable[[], AsyncIterator[Any]]): A nullary asynchronous function, - apply arguments with `functools.partial` or an anonymous function (lambda)
-            that returns an asynchronous iterator. This function is executed under the semaphore.
-
-    Yields:
-        Any: The items yielded by the asynchronous iterator function.
-
-    Usage:
-        ```python
-        import asyncio
-
-        async def async_iterator_function(param1, param2):
-            # Your async code here
-            yield ...
-
-        async def async_generator_function():
-            # Your async code to retrieve items
-            param1 = "your_param1"
-            yield param1
-
-        async def main():
-            semaphore = asyncio.BoundedSemaphore(50)
-            param2 = "your_param2"
-
-            tasks = [
-                semaphore_async_iterator(
-                    semaphore,
-                    lambda: async_iterator_function(param1, param2) # functools.partial(async_iterator_function, param1, param2)
-                )
-                async for param1 in async_generator_function()
-            ]
-
-            async for batch in stream_async_iterators_tasks(*tasks):
-                # Process each batch
-                pass
-
-        asyncio.run(main())
-        ```
-    """
-    async with semaphore:
-        async for result in function():
-            yield result

{port_ocean-0.12.2.dist-info → port_ocean-0.12.2.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.12.2
+Version: 0.12.2.dev2
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.12.2.dist-info → port_ocean-0.12.2.dev2.dist-info}/RECORD

@@ -65,7 +65,7 @@ port_ocean/core/defaults/clean.py,sha256=S3UAfca-oU89WJKIB4OgGjGjPr0vxBQ2aRZsLTZ
 port_ocean/core/defaults/common.py,sha256=uVUg6VEn4RqtXQwLwMNGfkmT5zYRN_h5USfKw3poVyo,3561
 port_ocean/core/defaults/initialize.py,sha256=qg4JLIWjp0c5-9w09X99muHRYX4k1cGZ_77vYo-tnpU,8422
 port_ocean/core/event_listener/__init__.py,sha256=mzJ33wRq0kh60fpVdOHVmvMTUQIvz3vxmifyBgwDn0E,889
-port_ocean/core/event_listener/base.py,sha256=1Nmpg00OfT2AD2L8eFm4VQEcdG2TClpSWJMhWhAjkEE,2356
+port_ocean/core/event_listener/base.py,sha256=tc7NHg-Vvt8M4iVIBrJvkecVTmTDGE1r-bjMQZpK_ME,2719
 port_ocean/core/event_listener/factory.py,sha256=AYYfSHPAF7P5H-uQECXT0JVJjKDHrYkWJJBSL4mGkg8,3697
 port_ocean/core/event_listener/http.py,sha256=N8HrfFqR3KGKz96pWdp_pP-m30jGtcz_1CkijovkBN8,2565
 port_ocean/core/event_listener/kafka.py,sha256=ulidnp4sz-chXwHsbH9JayVjcxy_mG6ts_Im3YKmLpI,6983
@@ -111,11 +111,11 @@ port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 port_ocean/helpers/async_client.py,sha256=SRlP6o7_FCSY3UHnRlZdezppePVxxOzZ0z861vE3K40,1783
 port_ocean/helpers/retry.py,sha256=IQ0RfQ2T5o6uoZh2WW2nrFH5TT6K_k3y2Im0HDp5j9Y,15059
 port_ocean/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-port_ocean/log/handlers.py,sha256=k9G_Mb4ga2-Jke9irpdlYqj6EYiwv0gEsh4TgyqqOmI,2853
+port_ocean/log/handlers.py,sha256=Br_1aujTmMkT15DbZj1E-mg0sHauvtKQB4E9-SDypmE,2861
 port_ocean/log/logger_setup.py,sha256=BaXt-mh9CVXhneh37H46d04lqOdIBixG1pFyGfotuZs,2328
 port_ocean/log/sensetive.py,sha256=lVKiZH6b7TkrZAMmhEJRhcl67HNM94e56x12DwFgCQk,2920
 port_ocean/middlewares.py,sha256=9wYCdyzRZGK1vjEJ28FY_DkfwDNENmXp504UKPf5NaQ,2727
-port_ocean/ocean.py,sha256=Oe4H3kKtkj52uNO4Rd_47iY3MBdrTtshXZ_16q7A8bM,5071
+port_ocean/ocean.py,sha256=ZPIRXqBF2WB3-tLsOwRnbFioGzGh2iRadK7YgYP1rpM,5227
 port_ocean/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/run.py,sha256=rTxBlrQd4yyrtgErCFJCHCEHs7d1OXrRiJehUYmIbN0,2212
 port_ocean/sonar-project.properties,sha256=X_wLzDOkEVmpGLRMb2fg9Rb0DxWwUFSvESId8qpvrPI,73
@@ -130,10 +130,9 @@ port_ocean/tests/helpers/ocean_app.py,sha256=Dp1bwEDhWsx_G-KVxOfJX1eVIS4168ajLu3
 port_ocean/tests/helpers/port_client.py,sha256=5d6GNr8vNNSOkrz1AdOhxBUKuusr_-UPDP7AVpHasQw,599
 port_ocean/tests/helpers/smoke_test.py,sha256=_9aJJFRfuGJEg2D2YQJVJRmpreS6gEPHHQq8Q01x4aQ,2697
 port_ocean/tests/test_smoke.py,sha256=uix2uIg_yOm8BHDgHw2hTFPy1fiIyxBGW3ENU_KoFlo,2557
-port_ocean/tests/utils/test_async_iterators.py,sha256=3PLk1emEXekb8LcC5GgVh3OicaX15i5WyaJT_eFnu_4,1336
 port_ocean/utils/__init__.py,sha256=KMGnCPXZJbNwtgxtyMycapkDz8tpSyw23MSYT3iVeHs,91
 port_ocean/utils/async_http.py,sha256=arnH458TExn2Dju_Sy6pHas_vF5RMWnOp-jBz5WAAcE,1226
-port_ocean/utils/async_iterators.py,sha256=CPXskYWkhkZtAG-ducEwM8537t3z5usPEqXR9vcivzw,3715
+port_ocean/utils/async_iterators.py,sha256=iw3cUHxfQm3zUSPdw2FmSXDU8E1Ppnys4TGhswNuQ8s,1569
 port_ocean/utils/cache.py,sha256=3KItZDE2yVrbVDr-hoM8lNna8s2dlpxhP4ICdLjH4LQ,2231
 port_ocean/utils/misc.py,sha256=0q2cJ5psqxn_5u_56pT7vOVQ3shDM02iC1lzyWQ_zl0,2098
 port_ocean/utils/queue_utils.py,sha256=KWWl8YVnG-glcfIHhM6nefY-2sou_C6DVP1VynQwzB4,2762
@@ -141,8 +140,8 @@ port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,32
 port_ocean/utils/signal.py,sha256=K-6kKFQTltcmKDhtyZAcn0IMa3sUpOHGOAUdWKgx0_E,1369
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.12.2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.12.2.dist-info/METADATA,sha256=CB3srEnScX99emeWn9qiASGQ6hRn2KO2DiCEEi1pMXs,6614
-port_ocean-0.12.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-port_ocean-0.12.2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.12.2.dist-info/RECORD,,
+port_ocean-0.12.2.dev2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.12.2.dev2.dist-info/METADATA,sha256=pFOjjt273h5-4BlEdo99jM90SCsMmMZKUa-waqQIRLc,6619
+port_ocean-0.12.2.dev2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+port_ocean-0.12.2.dev2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.12.2.dev2.dist-info/RECORD,,

port_ocean/tests/utils/test_async_iterators.py

@@ -1,45 +0,0 @@
-from typing import Any, AsyncGenerator
-import asyncio
-from port_ocean.utils.async_iterators import semaphore_async_iterator
-import pytest
-
-
-@pytest.mark.asyncio
-async def test_semaphore_async_iterator() -> None:
-    max_concurrency = 5
-    semaphore = asyncio.BoundedSemaphore(max_concurrency)
-
-    concurrent_tasks = 0
-    max_concurrent_tasks = 0
-    lock = asyncio.Lock()  # Protect shared variables
-
-    num_tasks = 20
-
-    async def mock_function() -> AsyncGenerator[str, None]:
-        nonlocal concurrent_tasks, max_concurrent_tasks
-
-        async with lock:
-            concurrent_tasks += 1
-            if concurrent_tasks > max_concurrent_tasks:
-                max_concurrent_tasks = concurrent_tasks
-
-        await asyncio.sleep(0.1)
-        yield "result"
-
-        async with lock:
-            concurrent_tasks -= 1
-
-    async def consume_iterator(async_iterator: Any) -> None:
-        async for _ in async_iterator:
-            pass
-
-    tasks = [
-        consume_iterator(semaphore_async_iterator(semaphore, mock_function))
-        for _ in range(num_tasks)
-    ]
-    await asyncio.gather(*tasks)
-
-    assert (
-        max_concurrent_tasks <= max_concurrency
-    ), f"Max concurrent tasks {max_concurrent_tasks} exceeded semaphore limit {max_concurrency}"
-    assert concurrent_tasks == 0, "Not all tasks have completed"