port-ocean 0.5.5__py3-none-any.whl → 0.17.8__py3-none-any.whl

port_ocean/tests/utils/test_async_iterators.py

@@ -0,0 +1,45 @@
+from typing import Any, AsyncGenerator
+import asyncio
+from port_ocean.utils.async_iterators import semaphore_async_iterator
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_semaphore_async_iterator() -> None:
+    max_concurrency = 5
+    semaphore = asyncio.BoundedSemaphore(max_concurrency)
+
+    concurrent_tasks = 0
+    max_concurrent_tasks = 0
+    lock = asyncio.Lock()  # Protect shared variables
+
+    num_tasks = 20
+
+    async def mock_function() -> AsyncGenerator[str, None]:
+        nonlocal concurrent_tasks, max_concurrent_tasks
+
+        async with lock:
+            concurrent_tasks += 1
+            if concurrent_tasks > max_concurrent_tasks:
+                max_concurrent_tasks = concurrent_tasks
+
+        await asyncio.sleep(0.1)
+        yield "result"
+
+        async with lock:
+            concurrent_tasks -= 1
+
+    async def consume_iterator(async_iterator: Any) -> None:
+        async for _ in async_iterator:
+            pass
+
+    tasks = [
+        consume_iterator(semaphore_async_iterator(semaphore, mock_function))
+        for _ in range(num_tasks)
+    ]
+    await asyncio.gather(*tasks)
+
+    assert (
+        max_concurrent_tasks <= max_concurrency
+    ), f"Max concurrent tasks {max_concurrent_tasks} exceeded semaphore limit {max_concurrency}"
+    assert concurrent_tasks == 0, "Not all tasks have completed"

port_ocean/tests/utils/test_cache.py

@@ -0,0 +1,189 @@
+from typing import Any
+import asyncio
+from port_ocean.utils import cache  # Import the module where 'event' is used
+import pytest
+from dataclasses import dataclass, field
+from typing import AsyncGenerator, AsyncIterator, List, TypeVar
+
+
+@dataclass
+class EventContext:
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+
+@pytest.fixture
+def event() -> EventContext:
+    return EventContext()
+
+
+T = TypeVar("T")
+
+
+async def collect_iterator_results(iterator: AsyncIterator[List[T]]) -> List[T]:
+    results = []
+    async for item in iterator:
+        results.extend(item)
+    return results
+
+
+@pytest.mark.asyncio
+async def test_cache_iterator_result(event: EventContext, monkeypatch: Any) -> None:
+    monkeypatch.setattr(cache, "event", event)
+
+    call_count = 0
+
+    @cache.cache_iterator_result()
+    async def sample_iterator(x: int) -> AsyncGenerator[List[int], None]:
+        nonlocal call_count
+        call_count += 1
+        for i in range(x):
+            await asyncio.sleep(0.1)
+            yield [i]
+
+    result1 = await collect_iterator_results(sample_iterator(3))
+    assert result1 == [0, 1, 2]
+    assert call_count == 1
+
+    result2 = await collect_iterator_results(sample_iterator(3))
+    assert result2 == [0, 1, 2]
+    assert call_count == 1
+
+    result3 = await collect_iterator_results(sample_iterator(4))
+    assert result3 == [0, 1, 2, 3]
+    assert call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_cache_iterator_result_with_kwargs(
+    event: EventContext, monkeypatch: Any
+) -> None:
+    monkeypatch.setattr(cache, "event", event)
+
+    call_count = 0
+
+    @cache.cache_iterator_result()
+    async def sample_iterator(x: int, y: int = 1) -> AsyncGenerator[List[int], None]:
+        nonlocal call_count
+        call_count += 1
+        for i in range(x * y):
+            await asyncio.sleep(0.1)
+            yield [i]
+
+    result1 = await collect_iterator_results(sample_iterator(2, y=2))
+    assert result1 == [0, 1, 2, 3]
+    assert call_count == 1
+
+    result2 = await collect_iterator_results(sample_iterator(2, y=2))
+    assert result2 == [0, 1, 2, 3]
+    assert call_count == 1
+
+    result3 = await collect_iterator_results(sample_iterator(2, y=3))
+    assert result3 == [0, 1, 2, 3, 4, 5]
+    assert call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_cache_iterator_result_no_cache(
+    event: EventContext, monkeypatch: Any
+) -> None:
+    monkeypatch.setattr(cache, "event", event)
+
+    call_count = 0
+
+    @cache.cache_iterator_result()
+    async def sample_iterator(x: int) -> AsyncGenerator[List[int], None]:
+        nonlocal call_count
+        call_count += 1
+        for i in range(x):
+            await asyncio.sleep(0.1)
+            yield [i]
+
+    result1 = await collect_iterator_results(sample_iterator(3))
+    assert result1 == [0, 1, 2]
+    assert call_count == 1
+
+    event.attributes.clear()
+
+    result2 = await collect_iterator_results(sample_iterator(3))
+    assert result2 == [0, 1, 2]
+    assert call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_cache_coroutine_result(event: EventContext, monkeypatch: Any) -> None:
+    monkeypatch.setattr(cache, "event", event)
+
+    call_count = 0
+
+    @cache.cache_coroutine_result()
+    async def sample_coroutine(x: int) -> int:
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.1)
+        return x * 2
+
+    result1 = await sample_coroutine(2)
+    assert result1 == 4
+    assert call_count == 1
+
+    result2 = await sample_coroutine(2)
+    assert result2 == 4
+    assert call_count == 1
+
+    result3 = await sample_coroutine(3)
+    assert result3 == 6
+    assert call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_cache_coroutine_result_with_kwargs(
+    event: EventContext, monkeypatch: Any
+) -> None:
+    monkeypatch.setattr(cache, "event", event)
+
+    call_count = 0
+
+    @cache.cache_coroutine_result()
+    async def sample_coroutine(x: int, y: int = 1) -> int:
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.1)
+        return x * y
+
+    result1 = await sample_coroutine(2, y=3)
+    assert result1 == 6
+    assert call_count == 1
+
+    result2 = await sample_coroutine(2, y=3)
+    assert result2 == 6
+    assert call_count == 1
+
+    result3 = await sample_coroutine(2, y=4)
+    assert result3 == 8
+    assert call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_cache_coroutine_result_no_cache(
+    event: EventContext, monkeypatch: Any
+) -> None:
+    monkeypatch.setattr(cache, "event", event)
+
+    call_count = 0
+
+    @cache.cache_coroutine_result()
+    async def sample_coroutine(x: int) -> int:
+        nonlocal call_count
+        call_count += 1
+        await asyncio.sleep(0.1)
+        return x * 2
+
+    result1 = await sample_coroutine(2)
+    assert result1 == 4
+    assert call_count == 1
+
+    event.attributes.clear()
+
+    result2 = await sample_coroutine(2)
+    assert result2 == 4
+    assert call_count == 2

port_ocean/utils/async_iterators.py

@@ -0,0 +1,109 @@
+import typing
+
+import aiostream
+
+if typing.TYPE_CHECKING:
+    from asyncio import Semaphore
+
+
+async def stream_async_iterators_tasks(
+    *tasks: typing.AsyncIterable[typing.Any],
+) -> typing.AsyncIterable[typing.Any]:
+    """
+    This function takes a list of async iterators and streams the results of each iterator as they are available.
+    By using this function you can combine multiple async iterators into a single stream of results, instead of waiting
+    for each iterator to finish before starting the next one.
+
+    Usage:
+    ```python
+    async def async_iterator1():
+        for i in range(10):
+            yield i
+            await asyncio.sleep(1)
+
+    async def async_iterator2():
+        for i in range(10, 20):
+            yield i
+            await asyncio.sleep(1)
+
+    async def main():
+        async for result in stream_async_iterators_tasks([async_iterator1(), async_iterator2()]):
+            print(result)
+    ```
+
+    Caution - Before using this function, make sure that the third-party API you are calling allows the number of
+    concurrent requests you are making. If the API has a rate limit, you may need to adjust the number of concurrent
+    requests to avoid hitting the rate limit.
+
+    :param tasks: A list of async iterators
+    :return: A stream of results
+    """
+    if not tasks:
+        return
+
+    if len(tasks) == 1:
+        async for batch_items in tasks[0]:
+            yield batch_items
+        return
+
+    combine = aiostream.stream.merge(tasks[0], *tasks[1:])
+    async with combine.stream() as streamer:
+        async for batch_items in streamer:
+            yield batch_items
+
+
+async def semaphore_async_iterator(
+    semaphore: "Semaphore",
+    function: typing.Callable[[], typing.AsyncIterator[typing.Any]],
+) -> typing.AsyncIterator[typing.Any]:
+    """
+    Executes an asynchronous iterator function under a semaphore to limit concurrency.
+
+    This function ensures that the provided asynchronous iterator function is executed
+    while respecting the concurrency limit imposed by the semaphore. It acquires the
+    semaphore before executing the function and releases it after the function completes,
+    thus controlling the number of concurrent executions.
+
+    Parameters:
+        semaphore (asyncio.Semaphore | asyncio.BoundedSemaphore): The semaphore used to limit concurrency.
+        function (Callable[[], AsyncIterator[Any]]): A nullary asynchronous function, - apply arguments with `functools.partial` or an anonymous function (lambda)
+            that returns an asynchronous iterator. This function is executed under the semaphore.
+
+    Yields:
+        Any: The items yielded by the asynchronous iterator function.
+
+    Usage:
+        ```python
+        import asyncio
+
+        async def async_iterator_function(param1, param2):
+            # Your async code here
+            yield ...
+
+        async def async_generator_function():
+            # Your async code to retrieve items
+            param1 = "your_param1"
+            yield param1
+
+        async def main():
+            semaphore = asyncio.BoundedSemaphore(50)
+            param2 = "your_param2"
+
+            tasks = [
+                semaphore_async_iterator(
+                    semaphore,
+                    lambda: async_iterator_function(param1, param2) # functools.partial(async_iterator_function, param1, param2)
+                )
+                async for param1 in async_generator_function()
+            ]
+
+            async for batch in stream_async_iterators_tasks(*tasks):
+                # Process each batch
+                pass
+
+        asyncio.run(main())
+        ```
+    """
+    async with semaphore:
+        async for result in function():
+            yield result

port_ocean/utils/cache.py

@@ -1,9 +1,10 @@
 import functools
 import hashlib
-from typing import Callable, AsyncIterator, Any
+from typing import Callable, AsyncIterator, Awaitable, Any
 from port_ocean.context.event import event
 
 AsyncIteratorCallable = Callable[..., AsyncIterator[list[Any]]]
+AsyncCallable = Callable[..., Awaitable[Any]]
 
 
 def hash_func(function_name: str, *args: Any, **kwargs: Any) -> str:
@@ -59,3 +60,38 @@ def cache_iterator_result() -> Callable[[AsyncIteratorCallable], AsyncIteratorCa
         return wrapper
 
     return decorator
+
+
+def cache_coroutine_result() -> Callable[[AsyncCallable], AsyncCallable]:
+    """Coroutine version of `cache_iterator_result` from port_ocean.utils.cache
+
+    Decorator that caches the result of a coroutine function.
+    It checks if the result is already in the cache, and if not,
+    fetches the result, caches it, and returns the cached value.
+
+    The cache is stored in the scope of the running event and is
+    removed when the event is finished.
+
+    Usage:
+    ```python
+    @cache_coroutine_result()
+    async def my_coroutine_function():
+        # Your code here
+    ```
+    """
+
+    def decorator(func: AsyncCallable) -> AsyncCallable:
+        @functools.wraps(func)
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            cache_key = hash_func(func.__name__, *args, **kwargs)
+
+            if cache := event.attributes.get(cache_key):
+                return cache
+
+            result = await func(*args, **kwargs)
+            event.attributes[cache_key] = result
+            return result
+
+        return wrapper
+
+    return decorator

port_ocean/utils/misc.py

@@ -1,3 +1,4 @@
+from enum import Enum
 import inspect
 from importlib.util import spec_from_file_location, module_from_spec
 from pathlib import Path
@@ -5,11 +6,16 @@ from time import time
 from types import ModuleType
 from typing import Callable, Any
 from uuid import uuid4
-
 import tomli
 import yaml
 
 
+class IntegrationStateStatus(Enum):
+    Running = "running"
+    Failed = "failed"
+    Completed = "completed"
+
+
 def get_time(seconds_precision: bool = True) -> float:
     """Return current time as Unix/Epoch timestamp, in seconds.
     :param seconds_precision: if True, return with seconds precision as integer (default).
@@ -29,13 +35,25 @@ def get_function_location(func: Callable[..., Any]) -> str:
     return f"{file_path}:{line_number}"
 
 
-def get_integration_version() -> str:
+def get_pyproject_data() -> dict[str, Any] | None:
     try:
         with open("./pyproject.toml", "rb") as toml_file:
             pyproject_data = tomli.load(toml_file)
-            return pyproject_data["tool"]["poetry"]["version"]
+            return pyproject_data["tool"]["poetry"]
     except (FileNotFoundError, KeyError):
-        return ""
+        return None
+
+
+def get_integration_version() -> str:
+    if data := get_pyproject_data():
+        return data["version"]
+    return ""
+
+
+def get_integration_name() -> str:
+    if data := get_pyproject_data():
+        return data["name"]
+    return ""
 
 
 def get_spec_file(path: Path = Path(".")) -> dict[str, Any] | None:

port_ocean/utils/queue_utils.py

@@ -0,0 +1,88 @@
+import asyncio
+from asyncio import Queue, Task
+from typing import Any, TypeVar, Callable, Coroutine
+
+from loguru import logger
+
+T = TypeVar("T")
+
+
+async def _start_processor_worker(
+    queue: Queue[Any | None],
+    func: Callable[..., Coroutine[Any, Any, T]],
+    results: list[T],
+    errors: list[Exception],
+) -> None:
+    while True:
+        try:
+            raw_params = await queue.get()
+            if raw_params is None:
+                return
+            logger.debug("Processing async task")
+            results.append(await func(*raw_params))
+        except Exception as e:
+            logger.error(f"Error processing task: {e}")
+            errors.append(e)
+        finally:
+            queue.task_done()
+
+
+async def process_in_queue(
+    objects_to_process: list[Any],
+    func: Callable[..., Coroutine[Any, Any, T]],
+    *args: Any,
+    concurrency: int = 50,
+) -> list[T]:
+    """
+    This function executes multiple asynchronous tasks in a bounded way
+    (e.g. having 200 tasks to execute, while running only 20 concurrently),
+    to prevent overload and memory issues when dealing with large sets of data and tasks.
+    read more -> https://stackoverflow.com/questions/38831322/making-1-milion-requests-with-aiohttp-asyncio-literally
+
+    Usage:
+    ```python
+    async def incrementBy(num: int, increment_by: int) -> int:
+        await asyncio.sleep(3)
+        return num + increment_by
+
+    async def main():
+        raw_objects = [1, 2, 3, 4, 5]
+        processed_objects = await process_in_queue(
+            raw_objects,
+            incrementBy,
+            5
+        )
+    ```
+
+    :param objects_to_process: A list of the raw objects to process
+    :param func: An async function that turns raw object into result object
+    :param args: Static arguments to pass to the func when called
+    :param concurrency: An integer specifying the concurrent workers count
+    :return: A list of result objects
+    """
+    queue: Queue[Any | None] = Queue(maxsize=concurrency * 2)
+    tasks: list[Task[Any]] = []
+    processing_results: list[T] = []
+    errors: list[Exception] = []
+
+    for i in range(concurrency):
+        tasks.append(
+            asyncio.create_task(
+                _start_processor_worker(queue, func, processing_results, errors)
+            )
+        )
+
+    for i in range(len(objects_to_process)):
+        await queue.put((objects_to_process[i], *args))
+
+    for i in range(concurrency):
+        # We put None value into the queue, so the workers will know that we
+        # are done sending more input and they can terminate
+        await queue.put(None)
+
+    await queue.join()
+    await asyncio.gather(*tasks)
+    if errors:
+        raise ExceptionGroup("Error processing tasks", errors)
+
+    return processing_results

port_ocean/utils/signal.py

@@ -1,4 +1,3 @@
-import signal
 from typing import Callable, Any
 
 from werkzeug.local import LocalProxy, LocalStack
@@ -13,10 +12,8 @@ from port_ocean.utils.misc import generate_uuid
 class SignalHandler:
     def __init__(self) -> None:
         self._handlers: dict[str, Callable[[], Any]] = {}
-        signal.signal(signal.SIGINT, lambda _, __: self._exit())
-        signal.signal(signal.SIGTERM, lambda _, __: self._exit())
 
-    def _exit(self) -> None:
+    def exit(self) -> None:
         """
         Handles the exit signal.
         """

port_ocean/utils/time.py

@@ -0,0 +1,54 @@
+import datetime
+from loguru import logger
+
+
+def convert_str_to_utc_datetime(time_str: str) -> datetime.datetime | None:
+    """
+    Convert a string representing time to a datetime object.
+    :param time_str: a string representing time in the format "2021-09-01T12:00:00Z"
+    """
+    aware_date = datetime.datetime.fromisoformat(time_str)
+    if time_str.endswith("Z"):
+        aware_date = datetime.datetime.fromisoformat(time_str.replace("Z", "+00:00"))
+    return aware_date.astimezone(datetime.timezone.utc)
+
+
+def convert_to_minutes(s: str) -> int:
+    minutes_per_unit = {"s": 1 / 60, "m": 1, "h": 60, "d": 1440, "w": 10080}
+    try:
+        return int(int(s[:-1]) * minutes_per_unit[s[-1]])
+    except Exception:
+        logger.error(f"Failed converting string to minutes, {s}")
+        raise ValueError(
+            f"Invalid format. Expected a string ending with {minutes_per_unit.keys()}"
+        )
+
+
+def get_next_occurrence(
+    interval_seconds: int,
+    start_time: datetime.datetime,
+    now: datetime.datetime | None = None,
+) -> datetime.datetime:
+    """
+    Predict the next occurrence of an event based on interval, start time, and current time.
+
+    :param interval_minutes: Interval between occurrences in minutes.
+    :param start_time: Start time of the event as a datetime object.
+    :param now: Current time as a datetime object.
+    :return: The next occurrence time as a datetime object.
+    """
+
+    if now is None:
+        now = datetime.datetime.now(tz=datetime.timezone.utc)
+    # Calculate the total seconds elapsed since the start time
+    elapsed_seconds = (now - start_time).total_seconds()
+
+    # Calculate the number of intervals that have passed
+    intervals_passed = int(elapsed_seconds // interval_seconds)
+
+    # Calculate the next occurrence time
+    next_occurrence = start_time + datetime.timedelta(
+        seconds=(intervals_passed + 1) * interval_seconds
+    )
+
+    return next_occurrence