port-ocean 0.12.2.dev22__tar.gz → 0.12.4__tar.gz

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.12.2.dev22
+Version: 0.12.4
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -16,7 +16,6 @@ Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -26,7 +25,7 @@ Requires-Dist: aiostream (>=0.5.2,<0.7.0)
 Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
 Requires-Dist: confluent-kafka (>=2.1.1,<3.0.0)
 Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
-Requires-Dist: fastapi (>=0.100,<0.116)
+Requires-Dist: fastapi (>=0.100,<0.112)
 Requires-Dist: httpx (>=0.24.1,<0.28.0)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
 Requires-Dist: jq (>=1.8.0,<2.0.0)
@@ -40,7 +39,7 @@ Requires-Dist: rich (>=13.4.1,<14.0.0) ; extra == "cli"
 Requires-Dist: six (>=1.16.0,<2.0.0)
 Requires-Dist: tomli (>=2.0.1,<3.0.0)
 Requires-Dist: urllib3 (>=1.26.16,<3.0.0)
-Requires-Dist: uvicorn (>=0.22,<0.33)
+Requires-Dist: uvicorn (>=0.22,<0.31)
 Requires-Dist: werkzeug (>=2.3.4,<4.0.0)
 Project-URL: Repository, https://github.com/port-labs/Port-Ocean
 Description-Content-Type: text/markdown

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/cli/commands/new.py

@@ -1,23 +1,13 @@
 # -*- coding: utf-8 -*-
 
-import click
 import json
-from cookiecutter.main import cookiecutter  # type: ignore
 import os
 
-from port_ocean.cli.commands.main import cli_start, print_logo, console
-from port_ocean.cli.utils import cli_root_path
-
-
-def symlink_makefile(result: str, name: str) -> None:
-    infra_make_file = os.path.join(
-        os.path.dirname(result), "../integrations/_infra/Makefile"
-    )
-    target_link_make_file = os.path.join(
-        os.path.dirname(result), f"../integrations/{name}/Makefile"
-    )
+import click
+from cookiecutter.main import cookiecutter  # type: ignore
 
-    os.link(infra_make_file, target_link_make_file)
+from port_ocean.cli.commands.main import cli_start, console, print_logo
+from port_ocean.cli.utils import cli_root_path
 
 
 def add_vscode_configuration(result: str, name: str) -> None:
@@ -73,11 +63,13 @@ def new(path: str, is_private_integration: bool) -> None:
             "is_private_integration": is_private_integration,
         },
     )
+
     name = result.split("/")[-1]
 
-    if not is_private_integration:
+    final_private_integration = os.path.exists(os.path.join(result, "Dockerfile"))
+
+    if not final_private_integration:
         add_vscode_configuration(result, name)
-        symlink_makefile(result, name)
 
     console.print(
         "\n🌊 Ahoy, Captain! Your project is ready to set sail into the vast ocean of possibilities!",
@@ -95,7 +87,7 @@ def new(path: str, is_private_integration: bool) -> None:
         "⚓️ Set sail with [blue]Ocean[/blue]: Run [bold][blue]ocean sail[/blue] <path_to_integration>[/bold] to run the project using Ocean.\n"
         f"▶️ [bold][blue]ocean sail {path}/{name}[/blue][/bold] \n"
     )
-    if not is_private_integration:
+    if not final_private_integration:
         console.print(
             "⚓️ Smooth sailing with [blue]Make[/blue]: Alternatively, you can run [bold][blue]make run[/blue][/bold] to launch your project using Make. \n"
             f"▶️ [bold][blue]make run {path}/{name}[/blue][/bold]"

port_ocean-0.12.4/port_ocean/cli/cookiecutter/hooks/post_gen_project.py

@@ -0,0 +1,20 @@
+import os
+import shutil
+
+
+def handle_private_integration_flags():
+    infra_make_file = "../_infra/Makefile"
+    target_link_make_file = os.path.join("./Makefile")
+
+    if "{{ cookiecutter.is_private_integration }}" == "True":
+        shutil.copyfile(infra_make_file, target_link_make_file)
+        os.remove("sonar-project.properties")
+        return
+
+    os.symlink(infra_make_file, target_link_make_file)
+    os.remove("Dockerfile")
+    os.remove(".dockerignore")
+
+
+if __name__ == "__main__":
+    handle_private_integration_flags()

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/context/ocean.py

@@ -1,7 +1,7 @@
 from typing import Callable, TYPE_CHECKING, Any, Literal, Union
 
+from fastapi import APIRouter
 from pydantic.main import BaseModel
-from starlette.routing import Router
 from werkzeug.local import LocalProxy
 
 from port_ocean.clients.port.types import UserAgentType
@@ -45,7 +45,7 @@ class PortOceanContext:
         return self.app.config
 
     @property
-    def router(self) -> Router:
+    def router(self) -> APIRouter:
         return self.app.integration_router
 
     @property

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/core/integrations/base.py

@@ -77,6 +77,5 @@ class BaseIntegration(SyncRawMixin, SyncMixin):
             )
 
         logger.info("Initializing event listener")
-        # event_listener = await self.event_listener_factory.create_event_listener()
-        # await event_listener.start()
-
+        event_listener = await self.event_listener_factory.create_event_listener()
+        await event_listener.start()

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -3,6 +3,7 @@ import inspect
 import typing
 from typing import Callable, Awaitable, Any
 
+import httpx
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -172,7 +173,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         self, resource_config: ResourceConfig, user_agent_type: UserAgentType
     ) -> tuple[list[Entity], list[Exception]]:
         results, errors = await self._get_resource_raw_results(resource_config)
-        passed_entities = []
         async_generators: list[ASYNC_GENERATOR_RESYNC_TYPE] = []
         raw_results: RAW_RESULT = []
         for result in results:
@@ -201,14 +201,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                             0, send_raw_data_examples_amount - len(passed_entities)
                         )
 
-                    # entities, register_errors = await self._register_resource_raw(
-                    #     resource_config,
-                    #     items,
-                    #     user_agent_type,
-                    #     send_raw_data_examples_amount=send_raw_data_examples_amount,
-                    # )
-                    # errors.extend(register_errors)
-                    # passed_entities.extend(entities.passed)
+                    entities, register_errors = await self._register_resource_raw(
+                        resource_config,
+                        items,
+                        user_agent_type,
+                        send_raw_data_examples_amount=send_raw_data_examples_amount,
+                    )
+                    errors.extend(register_errors)
+                    passed_entities.extend(entities.passed)
             except* OceanAbortException as error:
                 errors.append(error)
 
@@ -426,20 +426,20 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 use_cache=False
             )
             logger.info(f"Resync will use the following mappings: {app_config.dict()}")
-            # try:
-            #     did_fetched_current_state = True
-            #     entities_at_port = await ocean.port_client.search_entities(
-            #         user_agent_type
-            #     )
-            # except httpx.HTTPError as e:
-            #     logger.warning(
-            #         "Failed to fetch the current state of entities at Port. "
-            #         "Skipping delete phase due to unknown initial state. "
-            #         f"Error: {e}\n"
-            #         f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
-            #         f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
-            #     )
-            #     did_fetched_current_state = False
+            try:
+                did_fetched_current_state = True
+                entities_at_port = await ocean.port_client.search_entities(
+                    user_agent_type
+                )
+            except httpx.HTTPError as e:
+                logger.warning(
+                    "Failed to fetch the current state of entities at Port. "
+                    "Skipping delete phase due to unknown initial state. "
+                    f"Error: {e}\n"
+                    f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                    f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                )
+                did_fetched_current_state = False
 
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
@@ -458,36 +458,36 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
                 raise
-            # else:
-            #     if not did_fetched_current_state:
-            #         logger.warning(
-            #             "Due to an error before the resync, the previous state of entities at Port is unknown."
-            #             " Skipping delete phase due to unknown initial state."
-            #         )
-            #         return
-            #
-            #     logger.info("Starting resync diff calculation")
-            #     flat_created_entities, errors = zip_and_sum(creation_results) or [
-            #         [],
-            #         [],
-            #     ]
-            #
-            #     if errors:
-            #         message = f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state"
-            #         error_group = ExceptionGroup(
-            #             f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state",
-            #             errors,
-            #         )
-            #         if not silent:
-            #             raise error_group
-            #
-            #         logger.error(message, exc_info=error_group)
-            #     else:
-            #         logger.info(
-            #             f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
-            #         )
-            #         await self.entities_state_applier.delete_diff(
-            #             {"before": entities_at_port, "after": flat_created_entities},
-            #             user_agent_type,
-            #         )
-            #         logger.info("Resync finished successfully")
+            else:
+                if not did_fetched_current_state:
+                    logger.warning(
+                        "Due to an error before the resync, the previous state of entities at Port is unknown."
+                        " Skipping delete phase due to unknown initial state."
+                    )
+                    return
+
+                logger.info("Starting resync diff calculation")
+                flat_created_entities, errors = zip_and_sum(creation_results) or [
+                    [],
+                    [],
+                ]
+
+                if errors:
+                    message = f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state"
+                    error_group = ExceptionGroup(
+                        f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state",
+                        errors,
+                    )
+                    if not silent:
+                        raise error_group
+
+                    logger.error(message, exc_info=error_group)
+                else:
+                    logger.info(
+                        f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
+                    )
+                    await self.entities_state_applier.delete_diff(
+                        {"before": entities_at_port, "after": flat_created_entities},
+                        user_agent_type,
+                    )
+                    logger.info("Resync finished successfully")

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/exceptions/api.py

@@ -1,7 +1,7 @@
 import abc
 
+from fastapi.responses import Response, PlainTextResponse
 from port_ocean.exceptions.base import BaseOceanException
-from starlette.responses import Response, PlainTextResponse
 
 
 class BaseAPIException(BaseOceanException, abc.ABC):

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/log/logger_setup.py

@@ -15,8 +15,8 @@ from port_ocean.utils.signal import signal_handler
 def setup_logger(level: LogLevelType, enable_http_handler: bool) -> None:
     logger.remove()
     _stdout_loguru_handler(level)
-    # if enable_http_handler:
-    #     _http_loguru_handler(level)
+    if enable_http_handler:
+        _http_loguru_handler(level)
 
 
 def _stdout_loguru_handler(level: LogLevelType) -> None:

port_ocean-0.12.4/port_ocean/middlewares.py

@@ -0,0 +1,73 @@
+from typing import Callable, Awaitable
+
+from fastapi import Request, Response
+from loguru import logger
+
+from port_ocean.exceptions.api import BaseAPIException, InternalServerException
+from .context.event import event_context, EventType
+from .context.ocean import ocean
+from .utils.misc import get_time, generate_uuid
+
+
+async def _handle_silently(
+    call_next: Callable[[Request], Awaitable[Response]], request: Request
+) -> Response:
+    response: Response
+    try:
+        if request.url.path.startswith("/integration"):
+            async with event_context(EventType.HTTP_REQUEST, trigger_type="request"):
+                await ocean.integration.port_app_config_handler.get_port_app_config()
+                response = await call_next(request)
+        else:
+            response = await call_next(request)
+
+    except BaseAPIException as ex:
+        response = ex.response()
+        if response.status_code < 500:
+            logger.bind(exception=str(ex)).info(
+                "Request did not succeed due to client-side error"
+            )
+        else:
+            logger.opt(exception=True).warning(
+                "Request did not succeed due to server-side error"
+            )
+
+    except Exception:
+        logger.opt(exception=True).error("Request failed due to unexpected error")
+        response = InternalServerException().response()
+
+    return response
+
+
+async def request_handler(
+    request: Request, call_next: Callable[[Request], Awaitable[Response]]
+) -> Response:
+    """Middleware used by FastAPI to process each request, featuring:
+
+    - Contextualize request logs with a unique Request ID (UUID4) for each unique request.
+    - Catch exceptions during the request handling. Translate custom API exceptions into responses,
+      or treat (and log) unexpected exceptions.
+    """
+    start_time = get_time(seconds_precision=False)
+    request_id = generate_uuid()
+
+    with logger.contextualize(request_id=request_id):
+        log_level = (
+            "DEBUG"
+            if request.url.path == "/docs" or request.url.path == "/openapi.json"
+            else "INFO"
+        )
+        logger.bind(url=str(request.url), method=request.method).log(
+            log_level, f"Request to {request.url.path} started"
+        )
+        response = await _handle_silently(call_next, request)
+
+        end_time = get_time(seconds_precision=False)
+        time_elapsed = round(end_time - start_time, 5)
+        response.headers["X-Request-ID"] = request_id
+        response.headers["X-Process-Time"] = str(time_elapsed)
+        logger.bind(
+            time_elapsed=time_elapsed, response_status=response.status_code
+        ).log(log_level, f"Request to {request.url.path} ended")
+
+        return response

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/ocean.py

@@ -1,18 +1,16 @@
 import asyncio
-import contextlib
 import sys
 import threading
+from contextlib import asynccontextmanager
 from typing import Callable, Any, Dict, AsyncIterator, Type
 
 from fastapi import FastAPI, APIRouter
 from loguru import logger
 from pydantic import BaseModel
-from starlette.applications import Starlette
-from starlette.middleware import Middleware
-from starlette.responses import JSONResponse
-from starlette.routing import Route, Mount, Router
 from starlette.types import Scope, Receive, Send
 
+from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
+from port_ocean.core.models import Runtime
 from port_ocean.clients.port.client import PortClient
 from port_ocean.config.settings import (
     IntegrationConfiguration,
@@ -22,29 +20,27 @@ from port_ocean.context.ocean import (
     ocean,
     initialize_port_ocean_context,
 )
-from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
 from port_ocean.core.integrations.base import BaseIntegration
-from port_ocean.core.models import Runtime
 from port_ocean.log.sensetive import sensitive_log_filter
-from port_ocean.middlewares import RequestHandlerMiddleware
+from port_ocean.middlewares import request_handler
 from port_ocean.utils.repeat import repeat_every
 from port_ocean.utils.signal import signal_handler
 from port_ocean.version import __integration_version__
+from port_ocean.utils.misc import IntegrationStateStatus
 
 
 class Ocean:
     def __init__(
         self,
-        app: Starlette | None = None,
+        app: FastAPI | None = None,
         integration_class: Callable[[PortOceanContext], BaseIntegration] | None = None,
-        integration_router: Router | None = None,
+        integration_router: APIRouter | None = None,
         config_factory: Type[BaseModel] | None = None,
         config_override: Dict[str, Any] | None = None,
     ):
         initialize_port_ocean_context(self)
-
-        self.starlette_app = app or Starlette()
-        self.integration_router = integration_router or Router()
+        self.fast_api_app = app or FastAPI()
+        self.fast_api_app.middleware("http")(request_handler)
 
         self.config = IntegrationConfiguration(
             # type: ignore
@@ -56,6 +52,7 @@ class Ocean:
         sensitive_log_filter.hide_sensitive_strings(
             *self.config.get_sensitive_fields_data()
         )
+        self.integration_router = integration_router or APIRouter()
 
         self.port_client = PortClient(
             base_url=self.config.port.base_url,
@@ -80,20 +77,20 @@ class Ocean:
         self,
     ) -> None:
         async def execute_resync_all() -> None:
-            # await self.resync_state_updater.update_before_resync()
+            await self.resync_state_updater.update_before_resync()
             logger.info("Starting a new scheduled resync")
             try:
                 await self.integration.sync_raw_all()
-                # await self.resync_state_updater.update_after_resync()
+                await self.resync_state_updater.update_after_resync()
             except asyncio.CancelledError:
                 logger.warning(
                     "resync was cancelled by the scheduled resync, skipping state update"
                 )
-            # except Exception as e:
-            #     await self.resync_state_updater.update_after_resync(
-            #         IntegrationStateStatus.Failed
-            #     )
-            #     raise e
+            except Exception as e:
+                await self.resync_state_updater.update_after_resync(
+                    IntegrationStateStatus.Failed
+                )
+                raise e
 
         interval = self.config.scheduled_resync_interval
         loop = asyncio.get_event_loop()
@@ -105,7 +102,7 @@ class Ocean:
             repeated_function = repeat_every(
                 seconds=interval * 60,
                 # Not running the resync immediately because the event listener should run resync on startup
-                wait_first=False,
+                wait_first=True,
             )(
                 lambda: threading.Thread(
                     target=lambda: asyncio.run_coroutine_threadsafe(
@@ -116,8 +113,10 @@ class Ocean:
             await repeated_function()
 
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        @contextlib.asynccontextmanager
-        async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+        self.fast_api_app.include_router(self.integration_router, prefix="/integration")
+
+        @asynccontextmanager
+        async def lifecycle(_: FastAPI) -> AsyncIterator[None]:
             try:
                 await self.integration.start()
                 await self._setup_scheduled_resync()
@@ -128,16 +127,5 @@ class Ocean:
             finally:
                 signal_handler.exit()
 
-        async def health() -> JSONResponse:
-            return JSONResponse({"ok": True})
-
-        self.starlette_app = Starlette(
-            routes=[
-                Route("/docs", endpoint=health),
-                Mount("/integration", routes=self.integration_router.routes),
-            ],
-            middleware=[Middleware(RequestHandlerMiddleware)],
-            lifespan=lifespan,
-        )
-
-        await self.starlette_app(scope, receive, send)
+        self.fast_api_app.router.lifespan_context = lifecycle
+        await self.fast_api_app(scope, receive, send)

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/tests/helpers/fixtures.py

@@ -1,5 +1,5 @@
 from os import path
-from typing import Any, Callable, List, Tuple
+from typing import Any, Callable, Dict, List, Tuple, Union
 
 import pytest
 import pytest_asyncio
@@ -30,8 +30,8 @@ def port_client_for_fake_integration() -> Tuple[SmokeTestDetails, PortClient]:
 def get_mocked_ocean_app(request: Any) -> Callable[[], Ocean]:
     test_dir = path.join(path.dirname(request.module.__file__), "..")
 
-    def get_ocean_app() -> Ocean:
-        return get_integration_ocean_app(test_dir)
+    def get_ocean_app(config_overrides: Union[Dict[str, Any], None] = None) -> Ocean:
+        return get_integration_ocean_app(test_dir, config_overrides)
 
     return get_ocean_app
 

{port_ocean-0.12.2.dev22 → port_ocean-0.12.4}/port_ocean/tests/helpers/ocean_app.py

@@ -1,7 +1,7 @@
 import sys
 from inspect import getmembers
 from pathlib import Path
-from typing import List, Tuple
+from typing import Any, Dict, List, Tuple, Union
 
 from yaml import safe_load
 
@@ -12,7 +12,9 @@ from port_ocean.ocean import Ocean
 from port_ocean.utils.misc import get_spec_file, load_module
 
 
-def get_integration_ocean_app(integration_path: str) -> Ocean:
+def get_integration_ocean_app(
+    integration_path: str, config_overrides: Union[Dict[str, Any], None] = None
+) -> Ocean:
     spec_file = get_spec_file(Path(integration_path))
 
     config_factory = None if not spec_file else spec_file.get("configurations", [])
@@ -21,9 +23,12 @@ def get_integration_ocean_app(integration_path: str) -> Ocean:
         integration_path,
         config_factory,
         {
-            "port": {
-                "client_id": "bla",
-                "client_secret": "bla",
+            **(config_overrides or {}),
+            **{
+                "port": {
+                    "client_id": "bla",
+                    "client_secret": "bla",
+                },
             },
         },
     )

port_ocean-0.12.4/port_ocean/tests/utils/test_async_iterators.py

@@ -0,0 +1,45 @@
+from typing import Any, AsyncGenerator
+import asyncio
+from port_ocean.utils.async_iterators import semaphore_async_iterator
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_semaphore_async_iterator() -> None:
+    max_concurrency = 5
+    semaphore = asyncio.BoundedSemaphore(max_concurrency)
+
+    concurrent_tasks = 0
+    max_concurrent_tasks = 0
+    lock = asyncio.Lock()  # Protect shared variables
+
+    num_tasks = 20
+
+    async def mock_function() -> AsyncGenerator[str, None]:
+        nonlocal concurrent_tasks, max_concurrent_tasks
+
+        async with lock:
+            concurrent_tasks += 1
+            if concurrent_tasks > max_concurrent_tasks:
+                max_concurrent_tasks = concurrent_tasks
+
+        await asyncio.sleep(0.1)
+        yield "result"
+
+        async with lock:
+            concurrent_tasks -= 1
+
+    async def consume_iterator(async_iterator: Any) -> None:
+        async for _ in async_iterator:
+            pass
+
+    tasks = [
+        consume_iterator(semaphore_async_iterator(semaphore, mock_function))
+        for _ in range(num_tasks)
+    ]
+    await asyncio.gather(*tasks)
+
+    assert (
+        max_concurrent_tasks <= max_concurrency
+    ), f"Max concurrent tasks {max_concurrent_tasks} exceeded semaphore limit {max_concurrency}"
+    assert concurrent_tasks == 0, "Not all tasks have completed"