port-ocean 0.12.2.dev23__tar.gz → 0.12.3__tar.gz

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.12.2.dev23
+Version: 0.12.3
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -16,7 +16,6 @@ Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -26,7 +25,7 @@ Requires-Dist: aiostream (>=0.5.2,<0.7.0)
 Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
 Requires-Dist: confluent-kafka (>=2.1.1,<3.0.0)
 Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
-Requires-Dist: fastapi (>=0.100,<0.116)
+Requires-Dist: fastapi (>=0.100,<0.112)
 Requires-Dist: httpx (>=0.24.1,<0.28.0)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
 Requires-Dist: jq (>=1.8.0,<2.0.0)
@@ -40,7 +39,7 @@ Requires-Dist: rich (>=13.4.1,<14.0.0) ; extra == "cli"
 Requires-Dist: six (>=1.16.0,<2.0.0)
 Requires-Dist: tomli (>=2.0.1,<3.0.0)
 Requires-Dist: urllib3 (>=1.26.16,<3.0.0)
-Requires-Dist: uvicorn (>=0.22,<0.33)
+Requires-Dist: uvicorn (>=0.22,<0.31)
 Requires-Dist: werkzeug (>=2.3.4,<4.0.0)
 Project-URL: Repository, https://github.com/port-labs/Port-Ocean
 Description-Content-Type: text/markdown

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/port_ocean/context/ocean.py

@@ -1,7 +1,7 @@
 from typing import Callable, TYPE_CHECKING, Any, Literal, Union
 
+from fastapi import APIRouter
 from pydantic.main import BaseModel
-from starlette.routing import Router
 from werkzeug.local import LocalProxy
 
 from port_ocean.clients.port.types import UserAgentType
@@ -45,7 +45,7 @@ class PortOceanContext:
         return self.app.config
 
     @property
-    def router(self) -> Router:
+    def router(self) -> APIRouter:
         return self.app.integration_router
 
     @property

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/port_ocean/core/integrations/base.py

@@ -77,6 +77,5 @@ class BaseIntegration(SyncRawMixin, SyncMixin):
             )
 
         logger.info("Initializing event listener")
-        # event_listener = await self.event_listener_factory.create_event_listener()
-        # await event_listener.start()
-
+        event_listener = await self.event_listener_factory.create_event_listener()
+        await event_listener.start()

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -3,6 +3,7 @@ import inspect
 import typing
 from typing import Callable, Awaitable, Any
 
+import httpx
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -172,7 +173,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         self, resource_config: ResourceConfig, user_agent_type: UserAgentType
     ) -> tuple[list[Entity], list[Exception]]:
         results, errors = await self._get_resource_raw_results(resource_config)
-        passed_entities = []
         async_generators: list[ASYNC_GENERATOR_RESYNC_TYPE] = []
         raw_results: RAW_RESULT = []
         for result in results:
@@ -201,14 +201,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                             0, send_raw_data_examples_amount - len(passed_entities)
                         )
 
-                    # entities, register_errors = await self._register_resource_raw(
-                    #     resource_config,
-                    #     items,
-                    #     user_agent_type,
-                    #     send_raw_data_examples_amount=send_raw_data_examples_amount,
-                    # )
-                    # errors.extend(register_errors)
-                    # passed_entities.extend(entities.passed)
+                    entities, register_errors = await self._register_resource_raw(
+                        resource_config,
+                        items,
+                        user_agent_type,
+                        send_raw_data_examples_amount=send_raw_data_examples_amount,
+                    )
+                    errors.extend(register_errors)
+                    passed_entities.extend(entities.passed)
             except* OceanAbortException as error:
                 errors.append(error)
 
@@ -426,20 +426,20 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 use_cache=False
             )
             logger.info(f"Resync will use the following mappings: {app_config.dict()}")
-            # try:
-            #     did_fetched_current_state = True
-            #     entities_at_port = await ocean.port_client.search_entities(
-            #         user_agent_type
-            #     )
-            # except httpx.HTTPError as e:
-            #     logger.warning(
-            #         "Failed to fetch the current state of entities at Port. "
-            #         "Skipping delete phase due to unknown initial state. "
-            #         f"Error: {e}\n"
-            #         f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
-            #         f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
-            #     )
-            #     did_fetched_current_state = False
+            try:
+                did_fetched_current_state = True
+                entities_at_port = await ocean.port_client.search_entities(
+                    user_agent_type
+                )
+            except httpx.HTTPError as e:
+                logger.warning(
+                    "Failed to fetch the current state of entities at Port. "
+                    "Skipping delete phase due to unknown initial state. "
+                    f"Error: {e}\n"
+                    f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                    f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                )
+                did_fetched_current_state = False
 
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
@@ -458,36 +458,36 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
                 raise
-            # else:
-            #     if not did_fetched_current_state:
-            #         logger.warning(
-            #             "Due to an error before the resync, the previous state of entities at Port is unknown."
-            #             " Skipping delete phase due to unknown initial state."
-            #         )
-            #         return
-            #
-            #     logger.info("Starting resync diff calculation")
-            #     flat_created_entities, errors = zip_and_sum(creation_results) or [
-            #         [],
-            #         [],
-            #     ]
-            #
-            #     if errors:
-            #         message = f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state"
-            #         error_group = ExceptionGroup(
-            #             f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state",
-            #             errors,
-            #         )
-            #         if not silent:
-            #             raise error_group
-            #
-            #         logger.error(message, exc_info=error_group)
-            #     else:
-            #         logger.info(
-            #             f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
-            #         )
-            #         await self.entities_state_applier.delete_diff(
-            #             {"before": entities_at_port, "after": flat_created_entities},
-            #             user_agent_type,
-            #         )
-            #         logger.info("Resync finished successfully")
+            else:
+                if not did_fetched_current_state:
+                    logger.warning(
+                        "Due to an error before the resync, the previous state of entities at Port is unknown."
+                        " Skipping delete phase due to unknown initial state."
+                    )
+                    return
+
+                logger.info("Starting resync diff calculation")
+                flat_created_entities, errors = zip_and_sum(creation_results) or [
+                    [],
+                    [],
+                ]
+
+                if errors:
+                    message = f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state"
+                    error_group = ExceptionGroup(
+                        f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state",
+                        errors,
+                    )
+                    if not silent:
+                        raise error_group
+
+                    logger.error(message, exc_info=error_group)
+                else:
+                    logger.info(
+                        f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
+                    )
+                    await self.entities_state_applier.delete_diff(
+                        {"before": entities_at_port, "after": flat_created_entities},
+                        user_agent_type,
+                    )
+                    logger.info("Resync finished successfully")

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/port_ocean/exceptions/api.py

@@ -1,7 +1,7 @@
 import abc
 
+from fastapi.responses import Response, PlainTextResponse
 from port_ocean.exceptions.base import BaseOceanException
-from starlette.responses import Response, PlainTextResponse
 
 
 class BaseAPIException(BaseOceanException, abc.ABC):

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/port_ocean/log/logger_setup.py

@@ -15,8 +15,8 @@ from port_ocean.utils.signal import signal_handler
 def setup_logger(level: LogLevelType, enable_http_handler: bool) -> None:
     logger.remove()
     _stdout_loguru_handler(level)
-    # if enable_http_handler:
-    #     _http_loguru_handler(level)
+    if enable_http_handler:
+        _http_loguru_handler(level)
 
 
 def _stdout_loguru_handler(level: LogLevelType) -> None:

port_ocean-0.12.3/port_ocean/middlewares.py

@@ -0,0 +1,73 @@
+from typing import Callable, Awaitable
+
+from fastapi import Request, Response
+from loguru import logger
+
+from port_ocean.exceptions.api import BaseAPIException, InternalServerException
+from .context.event import event_context, EventType
+from .context.ocean import ocean
+from .utils.misc import get_time, generate_uuid
+
+
+async def _handle_silently(
+    call_next: Callable[[Request], Awaitable[Response]], request: Request
+) -> Response:
+    response: Response
+    try:
+        if request.url.path.startswith("/integration"):
+            async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
+                await ocean.integration.port_app_config_handler.get_port_app_config()
+                response = await call_next(request)
+        else:
+            response = await call_next(request)
+
+    except BaseAPIException as ex:
+        response = ex.response()
+        if response.status_code < 500:
+            logger.bind(exception=str(ex)).info(
+                "Request did not succeed due to client-side error"
+            )
+        else:
+            logger.opt(exception=True).warning(
+                "Request did not succeed due to server-side error"
+            )
+
+    except Exception:
+        logger.opt(exception=True).error("Request failed due to unexpected error")
+        response = InternalServerException().response()
+
+    return response
+
+
+async def request_handler(
+    request: Request, call_next: Callable[[Request], Awaitable[Response]]
+) -> Response:
+    """Middleware used by FastAPI to process each request, featuring:
+
+    - Contextualize request logs with a unique Request ID (UUID4) for each unique request.
+    - Catch exceptions during the request handling. Translate custom API exceptions into responses,
+      or treat (and log) unexpected exceptions.
+    """
+    start_time = get_time(seconds_precision=False)
+    request_id = generate_uuid()
+
+    with logger.contextualize(request_id=request_id):
+        log_level = (
+            "DEBUG"
+            if request.url.path == "/docs" or request.url.path == "/openapi.json"
+            else "INFO"
+        )
+        logger.bind(url=str(request.url), method=request.method).log(
+            log_level, f"Request to {request.url.path} started"
+        )
+        response = await _handle_silently(call_next, request)
+
+        end_time = get_time(seconds_precision=False)
+        time_elapsed = round(end_time - start_time, 5)
+        response.headers["X-Request-ID"] = request_id
+        response.headers["X-Process-Time"] = str(time_elapsed)
+        logger.bind(
+            time_elapsed=time_elapsed, response_status=response.status_code
+        ).log(log_level, f"Request to {request.url.path} ended")
+
+        return response

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/port_ocean/ocean.py

@@ -1,19 +1,16 @@
 import asyncio
-import contextlib
 import sys
 import threading
+from contextlib import asynccontextmanager
 from typing import Callable, Any, Dict, AsyncIterator, Type
 
 from fastapi import FastAPI, APIRouter
 from loguru import logger
 from pydantic import BaseModel
-from starlette.applications import Starlette
-from starlette.middleware import Middleware
-from starlette.requests import Request
-from starlette.responses import JSONResponse
-from starlette.routing import Route, Mount, Router
 from starlette.types import Scope, Receive, Send
 
+from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
+from port_ocean.core.models import Runtime
 from port_ocean.clients.port.client import PortClient
 from port_ocean.config.settings import (
     IntegrationConfiguration,
@@ -23,29 +20,27 @@ from port_ocean.context.ocean import (
     ocean,
     initialize_port_ocean_context,
 )
-from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
 from port_ocean.core.integrations.base import BaseIntegration
-from port_ocean.core.models import Runtime
 from port_ocean.log.sensetive import sensitive_log_filter
-from port_ocean.middlewares import RequestHandlerMiddleware
+from port_ocean.middlewares import request_handler
 from port_ocean.utils.repeat import repeat_every
 from port_ocean.utils.signal import signal_handler
 from port_ocean.version import __integration_version__
+from port_ocean.utils.misc import IntegrationStateStatus
 
 
 class Ocean:
     def __init__(
         self,
-        app: Starlette | None = None,
+        app: FastAPI | None = None,
         integration_class: Callable[[PortOceanContext], BaseIntegration] | None = None,
-        integration_router: Router | None = None,
+        integration_router: APIRouter | None = None,
         config_factory: Type[BaseModel] | None = None,
         config_override: Dict[str, Any] | None = None,
     ):
         initialize_port_ocean_context(self)
-
-        self.starlette_app = app or Starlette()
-        self.integration_router = integration_router or Router()
+        self.fast_api_app = app or FastAPI()
+        self.fast_api_app.middleware("http")(request_handler)
 
         self.config = IntegrationConfiguration(
             # type: ignore
@@ -57,6 +52,7 @@ class Ocean:
         sensitive_log_filter.hide_sensitive_strings(
             *self.config.get_sensitive_fields_data()
         )
+        self.integration_router = integration_router or APIRouter()
 
         self.port_client = PortClient(
             base_url=self.config.port.base_url,
@@ -81,20 +77,20 @@ class Ocean:
         self,
     ) -> None:
         async def execute_resync_all() -> None:
-            # await self.resync_state_updater.update_before_resync()
+            await self.resync_state_updater.update_before_resync()
             logger.info("Starting a new scheduled resync")
             try:
                 await self.integration.sync_raw_all()
-                # await self.resync_state_updater.update_after_resync()
+                await self.resync_state_updater.update_after_resync()
             except asyncio.CancelledError:
                 logger.warning(
                     "resync was cancelled by the scheduled resync, skipping state update"
                 )
-            # except Exception as e:
-            #     await self.resync_state_updater.update_after_resync(
-            #         IntegrationStateStatus.Failed
-            #     )
-            #     raise e
+            except Exception as e:
+                await self.resync_state_updater.update_after_resync(
+                    IntegrationStateStatus.Failed
+                )
+                raise e
 
         interval = self.config.scheduled_resync_interval
         loop = asyncio.get_event_loop()
@@ -106,7 +102,7 @@ class Ocean:
             repeated_function = repeat_every(
                 seconds=interval * 60,
                 # Not running the resync immediately because the event listener should run resync on startup
-                wait_first=False,
+                wait_first=True,
             )(
                 lambda: threading.Thread(
                     target=lambda: asyncio.run_coroutine_threadsafe(
@@ -117,8 +113,10 @@ class Ocean:
             await repeated_function()
 
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        @contextlib.asynccontextmanager
-        async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+        self.fast_api_app.include_router(self.integration_router, prefix="/integration")
+
+        @asynccontextmanager
+        async def lifecycle(_: FastAPI) -> AsyncIterator[None]:
             try:
                 await self.integration.start()
                 await self._setup_scheduled_resync()
@@ -129,16 +127,5 @@ class Ocean:
             finally:
                 signal_handler.exit()
 
-        async def health(request: Request) -> JSONResponse:
-            return JSONResponse({"ok": True})
-
-        self.starlette_app = Starlette(
-            routes=[
-                Route("/docs", endpoint=health),
-                Mount("/integration", routes=self.integration_router.routes),
-            ],
-            middleware=[Middleware(RequestHandlerMiddleware)],
-            lifespan=lifespan,
-        )
-
-        await self.starlette_app(scope, receive, send)
+        self.fast_api_app.router.lifespan_context = lifecycle
+        await self.fast_api_app(scope, receive, send)

port_ocean-0.12.3/port_ocean/tests/utils/test_async_iterators.py

@@ -0,0 +1,45 @@
+from typing import Any, AsyncGenerator
+import asyncio
+from port_ocean.utils.async_iterators import semaphore_async_iterator
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_semaphore_async_iterator() -> None:
+    max_concurrency = 5
+    semaphore = asyncio.BoundedSemaphore(max_concurrency)
+
+    concurrent_tasks = 0
+    max_concurrent_tasks = 0
+    lock = asyncio.Lock()  # Protect shared variables
+
+    num_tasks = 20
+
+    async def mock_function() -> AsyncGenerator[str, None]:
+        nonlocal concurrent_tasks, max_concurrent_tasks
+
+        async with lock:
+            concurrent_tasks += 1
+            if concurrent_tasks > max_concurrent_tasks:
+                max_concurrent_tasks = concurrent_tasks
+
+        await asyncio.sleep(0.1)
+        yield "result"
+
+        async with lock:
+            concurrent_tasks -= 1
+
+    async def consume_iterator(async_iterator: Any) -> None:
+        async for _ in async_iterator:
+            pass
+
+    tasks = [
+        consume_iterator(semaphore_async_iterator(semaphore, mock_function))
+        for _ in range(num_tasks)
+    ]
+    await asyncio.gather(*tasks)
+
+    assert (
+        max_concurrent_tasks <= max_concurrency
+    ), f"Max concurrent tasks {max_concurrent_tasks} exceeded semaphore limit {max_concurrency}"
+    assert concurrent_tasks == 0, "Not all tasks have completed"

port_ocean-0.12.3/port_ocean/utils/async_iterators.py

@@ -0,0 +1,109 @@
+import typing
+
+import aiostream
+
+if typing.TYPE_CHECKING:
+    from asyncio import Semaphore
+
+
+async def stream_async_iterators_tasks(
+    *tasks: typing.AsyncIterable[typing.Any],
+) -> typing.AsyncIterable[typing.Any]:
+    """
+    This function takes a list of async iterators and streams the results of each iterator as they are available.
+    By using this function you can combine multiple async iterators into a single stream of results, instead of waiting
+    for each iterator to finish before starting the next one.
+
+    Usage:
+    ```python
+    async def async_iterator1():
+        for i in range(10):
+            yield i
+            await asyncio.sleep(1)
+
+    async def async_iterator2():
+        for i in range(10, 20):
+            yield i
+            await asyncio.sleep(1)
+
+    async def main():
+        async for result in stream_async_iterators_tasks([async_iterator1(), async_iterator2()]):
+            print(result)
+    ```
+
+    Caution - Before using this function, make sure that the third-party API you are calling allows the number of
+    concurrent requests you are making. If the API has a rate limit, you may need to adjust the number of concurrent
+    requests to avoid hitting the rate limit.
+
+    :param tasks: A list of async iterators
+    :return: A stream of results
+    """
+    if not tasks:
+        return
+
+    if len(tasks) == 1:
+        async for batch_items in tasks[0]:
+            yield batch_items
+        return
+
+    combine = aiostream.stream.merge(tasks[0], *tasks[1:])
+    async with combine.stream() as streamer:
+        async for batch_items in streamer:
+            yield batch_items
+
+
+async def semaphore_async_iterator(
+    semaphore: "Semaphore",
+    function: typing.Callable[[], typing.AsyncIterator[typing.Any]],
+) -> typing.AsyncIterator[typing.Any]:
+    """
+    Executes an asynchronous iterator function under a semaphore to limit concurrency.
+
+    This function ensures that the provided asynchronous iterator function is executed
+    while respecting the concurrency limit imposed by the semaphore. It acquires the
+    semaphore before executing the function and releases it after the function completes,
+    thus controlling the number of concurrent executions.
+
+    Parameters:
+        semaphore (asyncio.Semaphore | asyncio.BoundedSemaphore): The semaphore used to limit concurrency.
+        function (Callable[[], AsyncIterator[Any]]): A nullary asynchronous function, - apply arguments with `functools.partial` or an anonymous function (lambda)
+            that returns an asynchronous iterator. This function is executed under the semaphore.
+
+    Yields:
+        Any: The items yielded by the asynchronous iterator function.
+
+    Usage:
+        ```python
+        import asyncio
+
+        async def async_iterator_function(param1, param2):
+            # Your async code here
+            yield ...
+
+        async def async_generator_function():
+            # Your async code to retrieve items
+            param1 = "your_param1"
+            yield param1
+
+        async def main():
+            semaphore = asyncio.BoundedSemaphore(50)
+            param2 = "your_param2"
+
+            tasks = [
+                semaphore_async_iterator(
+                    semaphore,
+                    lambda: async_iterator_function(param1, param2) # functools.partial(async_iterator_function, param1, param2)
+                )
+                async for param1 in async_generator_function()
+            ]
+
+            async for batch in stream_async_iterators_tasks(*tasks):
+                # Process each batch
+                pass
+
+        asyncio.run(main())
+        ```
+    """
+    async with semaphore:
+        async for result in function():
+            yield result

{port_ocean-0.12.2.dev23 → port_ocean-0.12.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.12.2-dev23"
+version = "0.12.3"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"
@@ -40,8 +40,8 @@ pydantic = { version = "^1.10.8", extras = ["dotenv"] }
 loguru = "^0.7.0"
 pyyaml = "^6.0"
 werkzeug = ">=2.3.4,<4.0.0"
-fastapi = ">=0.100,<0.116"
-uvicorn = ">=0.22,<0.33"
+fastapi = ">=0.100,<0.112"
+uvicorn = ">=0.22,<0.31"
 confluent-kafka = "^2.1.1"
 httpx = ">=0.24.1,<0.28.0"
 jq = "^1.8.0"