port-ocean 0.12.2__tar.gz → 0.12.2.dev2__tar.gz

{port_ocean-0.12.2 → port_ocean-0.12.2.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.12.2
+Version: 0.12.2.dev2
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.12.2 → port_ocean-0.12.2.dev2}/port_ocean/core/event_listener/base.py

@@ -1,12 +1,14 @@
+import asyncio
 from abc import abstractmethod
+from concurrent.futures import ProcessPoolExecutor
 from typing import TypedDict, Callable, Any, Awaitable
 
 from pydantic import Extra
 
 from port_ocean.config.base import BaseOceanModel
-from port_ocean.utils.signal import signal_handler
 from port_ocean.context.ocean import ocean
 from port_ocean.utils.misc import IntegrationStateStatus
+from port_ocean.utils.signal import signal_handler
 
 
 class EventListenerEvents(TypedDict):
@@ -67,7 +69,14 @@ class BaseEventListener:
         """
         await self._before_resync()
         try:
-            await self.events["on_resync"](resync_args)
+            loop = asyncio.get_event_loop()
+            with ProcessPoolExecutor() as executor:
+                e = executor.submit(
+                    lambda: asyncio.run_coroutine_threadsafe(
+                        self.events["on_resync"](resync_args), loop
+                    )
+                )
+                signal_handler.register(e.cancel)
             await self._after_resync()
         except Exception as e:
             await self._on_resync_failure(e)

{port_ocean-0.12.2 → port_ocean-0.12.2.dev2}/port_ocean/log/handlers.py

@@ -1,8 +1,8 @@
 import asyncio
 import logging
 import sys
-import threading
 import time
+from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from logging.handlers import MemoryHandler
 from typing import Any
@@ -65,18 +65,19 @@ class HTTPMemoryHandler(MemoryHandler):
         if self.ocean is None or not self.buffer:
             return
 
-        def _wrap_event_loop(_ocean: Ocean, logs_to_send: list[dict[str, Any]]) -> None:
-            loop = asyncio.new_event_loop()
-            loop.run_until_complete(self.send_logs(_ocean, logs_to_send))
-            loop.close()
-
         self.acquire()
         logs = list(self._serialized_buffer)
         if logs:
             self.buffer.clear()
             self._serialized_buffer.clear()
             self.last_flush_time = time.time()
-            threading.Thread(target=_wrap_event_loop, args=(self.ocean, logs)).start()
+            loop = asyncio.new_event_loop()
+            with ProcessPoolExecutor() as executor:
+                executor.submit(
+                    lambda: asyncio.run_coroutine_threadsafe(
+                        self.send_logs(self.ocean, logs), loop
+                    )
+                )
         self.release()
 
     async def send_logs(

{port_ocean-0.12.2 → port_ocean-0.12.2.dev2}/port_ocean/ocean.py

@@ -1,6 +1,6 @@
 import asyncio
 import sys
-import threading
+from concurrent.futures import ProcessPoolExecutor
 from contextlib import asynccontextmanager
 from typing import Callable, Any, Dict, AsyncIterator, Type
 
@@ -9,8 +9,6 @@ from loguru import logger
 from pydantic import BaseModel
 from starlette.types import Scope, Receive, Send
 
-from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
-from port_ocean.core.models import Runtime
 from port_ocean.clients.port.client import PortClient
 from port_ocean.config.settings import (
     IntegrationConfiguration,
@@ -20,13 +18,15 @@ from port_ocean.context.ocean import (
     ocean,
     initialize_port_ocean_context,
 )
+from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
 from port_ocean.core.integrations.base import BaseIntegration
+from port_ocean.core.models import Runtime
 from port_ocean.log.sensetive import sensitive_log_filter
 from port_ocean.middlewares import request_handler
+from port_ocean.utils.misc import IntegrationStateStatus
 from port_ocean.utils.repeat import repeat_every
 from port_ocean.utils.signal import signal_handler
 from port_ocean.version import __integration_version__
-from port_ocean.utils.misc import IntegrationStateStatus
 
 
 class Ocean:
@@ -92,6 +92,13 @@ class Ocean:
                 )
                 raise e
 
+        def pool_executor_wrapper(event_loop) -> None:
+            with ProcessPoolExecutor() as executor:
+                e = executor.submit(lambda: asyncio.run_coroutine_threadsafe(
+                    execute_resync_all(), event_loop
+                ))
+                signal_handler.register(e.cancel)
+
         interval = self.config.scheduled_resync_interval
         loop = asyncio.get_event_loop()
         if interval is not None:
@@ -103,13 +110,7 @@ class Ocean:
                 seconds=interval * 60,
                 # Not running the resync immediately because the event listener should run resync on startup
                 wait_first=True,
-            )(
-                lambda: threading.Thread(
-                    target=lambda: asyncio.run_coroutine_threadsafe(
-                        execute_resync_all(), loop
-                    )
-                ).start()
-            )
+            )(lambda _: pool_executor_wrapper(loop))
             await repeated_function()
 
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:

port_ocean-0.12.2.dev2/port_ocean/utils/async_iterators.py

@@ -0,0 +1,49 @@
+import typing
+
+import aiostream
+
+
+async def stream_async_iterators_tasks(
+    *tasks: typing.AsyncIterable[typing.Any],
+) -> typing.AsyncIterable[typing.Any]:
+    """
+    This function takes a list of async iterators and streams the results of each iterator as they are available.
+    By using this function you can combine multiple async iterators into a single stream of results, instead of waiting
+    for each iterator to finish before starting the next one.
+
+    Usage:
+    ```python
+    async def async_iterator1():
+        for i in range(10):
+            yield i
+            await asyncio.sleep(1)
+
+    async def async_iterator2():
+        for i in range(10, 20):
+            yield i
+            await asyncio.sleep(1)
+
+    async def main():
+        async for result in stream_async_iterators_tasks([async_iterator1(), async_iterator2()]):
+            print(result)
+    ```
+
+    Caution - Before using this function, make sure that the third-party API you are calling allows the number of
+    concurrent requests you are making. If the API has a rate limit, you may need to adjust the number of concurrent
+    requests to avoid hitting the rate limit.
+
+    :param tasks: A list of async iterators
+    :return: A stream of results
+    """
+    if not tasks:
+        return
+
+    if len(tasks) == 1:
+        async for batch_items in tasks[0]:
+            yield batch_items
+        return
+
+    combine = aiostream.stream.merge(tasks[0], *tasks[1:])
+    async with combine.stream() as streamer:
+        async for batch_items in streamer:
+            yield batch_items

{port_ocean-0.12.2 → port_ocean-0.12.2.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.12.2"
+version = "0.12.2-dev02"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"

port_ocean-0.12.2/port_ocean/tests/utils/test_async_iterators.py

@@ -1,45 +0,0 @@
-from typing import Any, AsyncGenerator
-import asyncio
-from port_ocean.utils.async_iterators import semaphore_async_iterator
-import pytest
-
-
-@pytest.mark.asyncio
-async def test_semaphore_async_iterator() -> None:
-    max_concurrency = 5
-    semaphore = asyncio.BoundedSemaphore(max_concurrency)
-
-    concurrent_tasks = 0
-    max_concurrent_tasks = 0
-    lock = asyncio.Lock()  # Protect shared variables
-
-    num_tasks = 20
-
-    async def mock_function() -> AsyncGenerator[str, None]:
-        nonlocal concurrent_tasks, max_concurrent_tasks
-
-        async with lock:
-            concurrent_tasks += 1
-            if concurrent_tasks > max_concurrent_tasks:
-                max_concurrent_tasks = concurrent_tasks
-
-        await asyncio.sleep(0.1)
-        yield "result"
-
-        async with lock:
-            concurrent_tasks -= 1
-
-    async def consume_iterator(async_iterator: Any) -> None:
-        async for _ in async_iterator:
-            pass
-
-    tasks = [
-        consume_iterator(semaphore_async_iterator(semaphore, mock_function))
-        for _ in range(num_tasks)
-    ]
-    await asyncio.gather(*tasks)
-
-    assert (
-        max_concurrent_tasks <= max_concurrency
-    ), f"Max concurrent tasks {max_concurrent_tasks} exceeded semaphore limit {max_concurrency}"
-    assert concurrent_tasks == 0, "Not all tasks have completed"

port_ocean-0.12.2/port_ocean/utils/async_iterators.py

@@ -1,109 +0,0 @@
-import typing
-
-import aiostream
-
-if typing.TYPE_CHECKING:
-    from asyncio import Semaphore
-
-
-async def stream_async_iterators_tasks(
-    *tasks: typing.AsyncIterable[typing.Any],
-) -> typing.AsyncIterable[typing.Any]:
-    """
-    This function takes a list of async iterators and streams the results of each iterator as they are available.
-    By using this function you can combine multiple async iterators into a single stream of results, instead of waiting
-    for each iterator to finish before starting the next one.
-
-    Usage:
-    ```python
-    async def async_iterator1():
-        for i in range(10):
-            yield i
-            await asyncio.sleep(1)
-
-    async def async_iterator2():
-        for i in range(10, 20):
-            yield i
-            await asyncio.sleep(1)
-
-    async def main():
-        async for result in stream_async_iterators_tasks([async_iterator1(), async_iterator2()]):
-            print(result)
-    ```
-
-    Caution - Before using this function, make sure that the third-party API you are calling allows the number of
-    concurrent requests you are making. If the API has a rate limit, you may need to adjust the number of concurrent
-    requests to avoid hitting the rate limit.
-
-    :param tasks: A list of async iterators
-    :return: A stream of results
-    """
-    if not tasks:
-        return
-
-    if len(tasks) == 1:
-        async for batch_items in tasks[0]:
-            yield batch_items
-        return
-
-    combine = aiostream.stream.merge(tasks[0], *tasks[1:])
-    async with combine.stream() as streamer:
-        async for batch_items in streamer:
-            yield batch_items
-
-
-async def semaphore_async_iterator(
-    semaphore: "Semaphore",
-    function: typing.Callable[[], typing.AsyncIterator[typing.Any]],
-) -> typing.AsyncIterator[typing.Any]:
-    """
-    Executes an asynchronous iterator function under a semaphore to limit concurrency.
-
-    This function ensures that the provided asynchronous iterator function is executed
-    while respecting the concurrency limit imposed by the semaphore. It acquires the
-    semaphore before executing the function and releases it after the function completes,
-    thus controlling the number of concurrent executions.
-
-    Parameters:
-        semaphore (asyncio.Semaphore | asyncio.BoundedSemaphore): The semaphore used to limit concurrency.
-        function (Callable[[], AsyncIterator[Any]]): A nullary asynchronous function, - apply arguments with `functools.partial` or an anonymous function (lambda)
-            that returns an asynchronous iterator. This function is executed under the semaphore.
-
-    Yields:
-        Any: The items yielded by the asynchronous iterator function.
-
-    Usage:
-        ```python
-        import asyncio
-
-        async def async_iterator_function(param1, param2):
-            # Your async code here
-            yield ...
-
-        async def async_generator_function():
-            # Your async code to retrieve items
-            param1 = "your_param1"
-            yield param1
-
-        async def main():
-            semaphore = asyncio.BoundedSemaphore(50)
-            param2 = "your_param2"
-
-            tasks = [
-                semaphore_async_iterator(
-                    semaphore,
-                    lambda: async_iterator_function(param1, param2) # functools.partial(async_iterator_function, param1, param2)
-                )
-                async for param1 in async_generator_function()
-            ]
-
-            async for batch in stream_async_iterators_tasks(*tasks):
-                # Process each batch
-                pass
-
-        asyncio.run(main())
-        ```
-    """
-    async with semaphore:
-        async for result in function():
-            yield result