port-ocean 0.5.6__py3-none-any.whl → 0.17.8__py3-none-any.whl

port_ocean/core/defaults/initialize.py

@@ -2,23 +2,26 @@ import asyncio
 from typing import Type, Any
 
 import httpx
-from starlette import status
 from loguru import logger
 
 from port_ocean.clients.port.client import PortClient
 from port_ocean.clients.port.types import UserAgentType
 from port_ocean.config.settings import IntegrationConfiguration
 from port_ocean.context.ocean import ocean
-from port_ocean.core.defaults.common import Defaults, get_port_integration_defaults
+from port_ocean.core.defaults.common import (
+    Defaults,
+    get_port_integration_defaults,
+)
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
 from port_ocean.core.models import Blueprint
+from port_ocean.core.utils.utils import gather_and_split_errors_from_results
 from port_ocean.exceptions.port_defaults import (
     AbortDefaultCreationError,
 )
 
 
 def deconstruct_blueprints_to_creation_steps(
-    raw_blueprints: list[dict[str, Any]]
+    raw_blueprints: list[dict[str, Any]],
 ) -> tuple[list[dict[str, Any]], ...]:
     """
     Deconstructing the blueprint into stages so the api wont fail to create a blueprint if there is a conflict
@@ -49,69 +52,98 @@ def deconstruct_blueprints_to_creation_steps(
     )
 
 
-async def _create_resources(
+async def _initialize_required_integration_settings(
     port_client: PortClient,
-    defaults: Defaults,
+    default_mapping: PortAppConfig,
     integration_config: IntegrationConfiguration,
 ) -> None:
-    response = await port_client._get_current_integration()
-    if response.status_code == status.HTTP_404_NOT_FOUND:
-        logger.info("Integration doesn't exist, creating new integration")
-    else:
-        logger.info("Integration already exists, skipping integration creation...")
-        return
+    try:
+        logger.info("Initializing integration at port")
+        integration = await port_client.get_current_integration(
+            should_log=False, should_raise=False
+        )
+        if not integration:
+            logger.info(
+                "Integration does not exist, Creating new integration with default mapping"
+            )
+            integration = await port_client.create_integration(
+                integration_config.integration.type,
+                integration_config.event_listener.get_changelog_destination_details(),
+                port_app_config=default_mapping,
+            )
+        elif not integration.get("config"):
+            logger.info(
+                "Encountered that the integration's mapping is empty, Initializing to default mapping"
+            )
+            integration = await port_client.patch_integration(
+                integration_config.integration.type,
+                integration_config.event_listener.get_changelog_destination_details(),
+                port_app_config=default_mapping,
+            )
+    except httpx.HTTPStatusError as err:
+        logger.error(f"Failed to apply default mapping: {err.response.text}.")
+        raise err
 
+    logger.info("Checking for diff in integration configuration")
+    changelog_destination = (
+        integration_config.event_listener.get_changelog_destination_details().get(
+            "changelog_destination"
+        )
+    )
+    if (
+        integration.get("changelogDestination") != changelog_destination
+        or integration.get("installationAppType") != integration_config.integration.type
+        or integration.get("version") != port_client.integration_version
+    ):
+        await port_client.patch_integration(
+            integration_config.integration.type, changelog_destination
+        )
+
+
+async def _create_resources(
+    port_client: PortClient,
+    defaults: Defaults,
+) -> None:
     creation_stage, *blueprint_patches = deconstruct_blueprints_to_creation_steps(
         defaults.blueprints
     )
 
-    blueprints_results = await asyncio.gather(
-        *(
+    blueprints_results, _ = await gather_and_split_errors_from_results(
+        [
             port_client.get_blueprint(blueprint["identifier"], should_log=False)
             for blueprint in creation_stage
-        ),
-        return_exceptions=True,
+        ],
+        lambda item: isinstance(item, Blueprint),
     )
 
-    existing_blueprints = [
-        result.identifier
-        for result in blueprints_results
-        if not isinstance(result, httpx.HTTPStatusError)
-        and isinstance(result, Blueprint)
-    ]
-
-    if existing_blueprints:
+    if blueprints_results:
         logger.info(
-            f"Blueprints already exist: {existing_blueprints}. Skipping integration default creation..."
+            f"Blueprints already exist: {[result.identifier for result in blueprints_results]}. Skipping integration default creation..."
         )
         return
 
-    create_results = await asyncio.gather(
-        *(
+    created_blueprints, blueprint_errors = await gather_and_split_errors_from_results(
+        (
             port_client.create_blueprint(
                 blueprint, user_agent_type=UserAgentType.exporter
             )
             for blueprint in creation_stage
-        ),
-        return_exceptions=True,
+        )
     )
 
-    errors = [result for result in create_results if isinstance(result, Exception)]
-    created_blueprints = [
-        result["identifier"]
-        for result in create_results
-        if not isinstance(result, BaseException)
-    ]
+    created_blueprints_identifiers = [bp["identifier"] for bp in created_blueprints]
 
-    if errors:
-        for error in errors:
+    if blueprint_errors:
+        for error in blueprint_errors:
             if isinstance(error, httpx.HTTPStatusError):
                 logger.warning(
                     f"Failed to create resources: {error.response.text}. Rolling back changes..."
                 )
 
-        raise AbortDefaultCreationError(created_blueprints, errors)
-    created_pages = []
+        raise AbortDefaultCreationError(
+            created_blueprints_identifiers, blueprint_errors
+        )
+
     try:
         for patch_stage in blueprint_patches:
             await asyncio.gather(
@@ -125,72 +157,67 @@ async def _create_resources(
                 )
             )
 
-        await asyncio.gather(
-            *(
-                port_client.create_action(blueprint_actions["blueprint"], action)
-                for blueprint_actions in defaults.actions
-                for action in blueprint_actions["data"]
+    except httpx.HTTPStatusError as err:
+        logger.error(f"Failed to create resources: {err.response.text}. continuing...")
+        raise AbortDefaultCreationError(created_blueprints_identifiers, [err])
+    try:
+        created_actions, actions_errors = await gather_and_split_errors_from_results(
+            (
+                port_client.create_action(action, should_log=False)
+                for action in defaults.actions
             )
         )
 
-        await asyncio.gather(
-            *(
-                port_client.create_scorecard(blueprint_scorecards["blueprint"], action)
-                for blueprint_scorecards in defaults.scorecards
-                for action in blueprint_scorecards["data"]
+        created_scorecards, scorecards_errors = (
+            await gather_and_split_errors_from_results(
+                (
+                    port_client.create_scorecard(
+                        blueprint_scorecards["blueprint"], action, should_log=False
+                    )
+                    for blueprint_scorecards in defaults.scorecards
+                    for action in blueprint_scorecards["data"]
+                )
             )
         )
 
-        create_pages_result = await asyncio.gather(
-            *(port_client.create_page(page) for page in defaults.pages),
-            return_exceptions=True,
+        created_pages, pages_errors = await gather_and_split_errors_from_results(
+            (port_client.create_page(page, should_log=False) for page in defaults.pages)
         )
 
-        created_pages = [
-            result.get("identifier", "")
-            for result in create_pages_result
-            if not isinstance(result, BaseException)
-        ]
-
-        pages_errors = [
-            result for result in create_pages_result if isinstance(result, Exception)
-        ]
-
-        if pages_errors:
-            for error in pages_errors:
+        errors = actions_errors + scorecards_errors + pages_errors
+        if errors:
+            for error in errors:
                 if isinstance(error, httpx.HTTPStatusError):
                     logger.warning(
-                        f"Failed to create resources: {error.response.text}. Rolling back changes..."
+                        f"Failed to create resource: {error.response.text}. continuing..."
                     )
 
-            raise AbortDefaultCreationError(
-                created_blueprints, pages_errors, created_pages
-            )
-
-        await port_client.create_integration(
-            integration_config.integration.type,
-            integration_config.event_listener.to_request(),
-            port_app_config=defaults.port_app_config,
-        )
-    except httpx.HTTPStatusError as err:
-        logger.error(
-            f"Failed to create resources: {err.response.text}. Rolling back changes..."
-        )
-        raise AbortDefaultCreationError(created_blueprints, [err], created_pages)
+    except Exception as err:
+        logger.error(f"Failed to create resources: {err}. continuing...")
 
 
 async def _initialize_defaults(
     config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
 ) -> None:
     port_client = ocean.port_client
-    defaults = get_port_integration_defaults(config_class)
+    defaults = get_port_integration_defaults(
+        config_class, integration_config.resources_path
+    )
     if not defaults:
-        logger.warning("No defaults found. Skipping...")
+        logger.warning("No defaults found. Skipping initialization...")
         return None
 
+    if defaults.port_app_config:
+        await _initialize_required_integration_settings(
+            port_client, defaults.port_app_config, integration_config
+        )
+
+    if not integration_config.initialize_port_resources:
+        return
+
     try:
         logger.info("Found default resources, starting creation process")
-        await _create_resources(port_client, defaults, integration_config)
+        await _create_resources(port_client, defaults)
     except AbortDefaultCreationError as e:
         logger.warning(
             f"Failed to create resources. Rolling back blueprints : {e.blueprints_to_rollback}"
@@ -205,28 +232,12 @@ async def _initialize_defaults(
                 for identifier in e.blueprints_to_rollback
             )
         )
-        if e.pages_to_rollback:
-            logger.warning(
-                f"Failed to create resources. Rolling back pages : {e.pages_to_rollback}"
-            )
-            await asyncio.gather(
-                *(
-                    port_client.delete_page(
-                        identifier,
-                    )
-                    for identifier in e.pages_to_rollback
-                )
-            )
-
         raise ExceptionGroup(str(e), e.errors)
 
 
 def initialize_defaults(
     config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
 ) -> None:
-    try:
-        asyncio.new_event_loop().run_until_complete(
-            _initialize_defaults(config_class, integration_config)
-        )
-    except Exception as e:
-        logger.debug(f"Failed to initialize defaults, skipping... Error: {e}")
+    asyncio.new_event_loop().run_until_complete(
+        _initialize_defaults(config_class, integration_config)
+    )

port_ocean/core/event_listener/__init__.py

@@ -16,12 +16,18 @@ from port_ocean.core.event_listener.once import (
     OnceEventListener,
 )
 
+from port_ocean.core.event_listener.webhooks_only import (
+    WebhooksOnlyEventListener,
+    WebhooksOnlyEventListenerSettings,
+)
+
 
 EventListenerSettingsType = (
     HttpEventListenerSettings
     | KafkaEventListenerSettings
     | PollingEventListenerSettings
     | OnceEventListenerSettings
+    | WebhooksOnlyEventListenerSettings
 )
 
 __all__ = [
@@ -34,4 +40,6 @@ __all__ = [
     "PollingEventListenerSettings",
     "OnceEventListener",
     "OnceEventListenerSettings",
+    "WebhooksOnlyEventListener",
+    "WebhooksOnlyEventListenerSettings",
 ]

port_ocean/core/event_listener/base.py

@@ -1,11 +1,12 @@
 from abc import abstractmethod
-from asyncio import Task
 from typing import TypedDict, Callable, Any, Awaitable
 
 from pydantic import Extra
 
 from port_ocean.config.base import BaseOceanModel
 from port_ocean.utils.signal import signal_handler
+from port_ocean.context.ocean import ocean
+from port_ocean.utils.misc import IntegrationStateStatus
 
 
 class EventListenerEvents(TypedDict):
@@ -22,7 +23,6 @@ class BaseEventListener:
         events: EventListenerEvents,
     ):
         self.events = events
-        self._tasks_to_close: list[Task[Any]] = []
 
     async def start(self) -> None:
         signal_handler.register(self._stop)
@@ -32,23 +32,62 @@ class BaseEventListener:
     async def _start(self) -> None:
         pass
 
-    def stop(self) -> None:
-        self._stop()
-        for task in self._tasks_to_close:
-            task.cancel()
-
     def _stop(self) -> None:
         """
         Can be used for event listeners that need cleanup before exiting.
         """
         pass
 
+    async def _before_resync(self) -> None:
+        """
+        Can be used for event listeners that need to perform some action before resync.
+        """
+        await ocean.app.resync_state_updater.update_before_resync()
+
+    async def _after_resync(self) -> None:
+        """
+        Can be used for event listeners that need to perform some action after resync.
+        """
+        await ocean.app.resync_state_updater.update_after_resync()
+
+    async def _on_resync_failure(self, e: Exception) -> None:
+        """
+        Can be used for event listeners that need to handle resync failures.
+        """
+        await ocean.app.resync_state_updater.update_after_resync(
+            IntegrationStateStatus.Failed
+        )
+
+    async def _resync(
+        self,
+        resync_args: dict[Any, Any],
+    ) -> None:
+        """
+        Triggers the "on_resync" event.
+        """
+        await self._before_resync()
+        try:
+            await self.events["on_resync"](resync_args)
+            await self._after_resync()
+        except Exception as e:
+            await self._on_resync_failure(e)
+            raise e
+
 
 class EventListenerSettings(BaseOceanModel, extra=Extra.allow):
     type: str
+    should_resync: bool = True
 
-    def to_request(self) -> dict[str, Any]:
+    def get_changelog_destination_details(self) -> dict[str, Any]:
         """
-        Converts the Settings object to a dictionary representation (request format).
+        Returns the changelog destination configuration for the event listener.
+        By default, returns an empty dict. Only KAFKA and WEBHOOK event listeners need to override this
+        to provide their specific changelog destination details.
+
+        Returns:
+            dict[str, Any]: The changelog destination configuration. For example:
+                - KAFKA returns {"type": "KAFKA"}
+                - WEBHOOK returns {"type": "WEBHOOK", "url": "https://example.com/resync"}
+                - Other event listeners return {}
         """
-        return {"type": self.type}
+        return {}

port_ocean/core/event_listener/factory.py

@@ -17,6 +17,10 @@ from port_ocean.core.event_listener.base import (
     BaseEventListener,
     EventListenerEvents,
 )
+from port_ocean.core.event_listener.webhooks_only import (
+    WebhooksOnlyEventListener,
+    WebhooksOnlyEventListenerSettings,
+)
 from port_ocean.exceptions.core import UnsupportedEventListenerTypeException
 
 
@@ -88,7 +92,11 @@ class EventListenerFactory:
                     config, OnceEventListenerSettings
                 ), assert_message.format(type(config))
                 event_listener = OnceEventListener(wrapped_events, config)
-
+            case "webhooks_only":
+                assert isinstance(
+                    config, WebhooksOnlyEventListenerSettings
+                ), assert_message.format(type(config))
+                event_listener = WebhooksOnlyEventListener(wrapped_events, config)
             case _:
                 raise UnsupportedEventListenerTypeException(
                     f"Event listener {_type} not supported"

port_ocean/core/event_listener/http.py

@@ -27,9 +27,17 @@ class HttpEventListenerSettings(EventListenerSettings):
     type: Literal["WEBHOOK"]
     app_host: AnyHttpUrl = Field(..., sensitive=True)
 
-    def to_request(self) -> dict[str, Any]:
+    def get_changelog_destination_details(self) -> dict[str, Any]:
+        """
+        Returns the changelog destination configuration for the webhook event listener.
+        For webhook event listeners, this specifies the URL where changelog events should be sent.
+
+        Returns:
+            dict[str, Any]: A dictionary with the webhook URL where changelog events should be sent,
+                           constructed by appending "/resync" to the app_host.
+        """
         return {
-            **super().to_request(),
+            "type": self.type,
             "url": self.app_host + "/resync",
         }
 
@@ -64,6 +72,6 @@ class HttpEventListener(BaseEventListener):
 
         @target_channel_router.post("/resync")
         async def resync() -> None:
-            await self.events["on_resync"]({})
+            await self._resync({})
 
         ocean.app.fast_api_app.include_router(target_channel_router)

port_ocean/core/event_listener/kafka.py

@@ -38,12 +38,27 @@ class KafkaEventListenerSettings(EventListenerSettings):
     """
 
     type: Literal["KAFKA"]
-    brokers: str = "b-1-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-2-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-3-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196"
+    brokers: str = (
+        "b-1-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-2-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-3-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196"
+    )
     security_protocol: str = "SASL_SSL"
     authentication_mechanism: str = "SCRAM-SHA-512"
     kafka_security_enabled: bool = True
     consumer_poll_timeout: int = 1
 
+    def get_changelog_destination_details(self) -> dict[str, Any]:
+        """
+        Returns the changelog destination configuration for the Kafka event listener.
+        For Kafka event listeners, this specifies that changelog events should be sent via Kafka.
+
+        Returns:
+            dict[str, Any]: A dictionary with type "KAFKA" to indicate that changelog events
+                           should be sent through the Kafka message bus.
+        """
+        return {
+            "type": self.type,
+        }
+
 
 class KafkaEventListener(BaseEventListener):
     """
@@ -97,9 +112,13 @@ class KafkaEventListener(BaseEventListener):
             return False
 
         integration_identifier = after.get("identifier")
-        if integration_identifier == self.integration_identifier and (
-            "change.log" in topic
-        ):
+        if integration_identifier != self.integration_identifier:
+            return False
+
+        if after.get("updatedAt") == after.get("resyncState", {}).get("updatedAt"):
+            return False
+
+        if "change.log" in topic:
             return msg_value.get("changelogDestination", {}).get("type", "") == "KAFKA"
 
         return False
@@ -120,7 +139,7 @@ class KafkaEventListener(BaseEventListener):
 
         if "change.log" in topic and message is not None:
             try:
-                await self.events["on_resync"](message)
+                await self._resync(message)
             except Exception as e:
                 _type, _, tb = sys.exc_info()
                 logger.opt(exception=(_type, None, tb)).error(

port_ocean/core/event_listener/once.py

@@ -1,3 +1,4 @@
+import datetime
 import signal
 from typing import Literal, Any
 
@@ -9,6 +10,9 @@ from port_ocean.core.event_listener.base import (
     EventListenerSettings,
 )
 from port_ocean.utils.repeat import repeat_every
+from port_ocean.context.ocean import ocean
+from port_ocean.utils.time import convert_str_to_utc_datetime, convert_to_minutes
+from port_ocean.utils.misc import IntegrationStateStatus
 
 
 class OnceEventListenerSettings(EventListenerSettings):
@@ -19,9 +23,6 @@ class OnceEventListenerSettings(EventListenerSettings):
 
     type: Literal["ONCE"]
 
-    def to_request(self) -> dict[str, Any]:
-        return {}
-
 
 class OnceEventListener(BaseEventListener):
     """
@@ -41,6 +42,97 @@ class OnceEventListener(BaseEventListener):
     ):
         super().__init__(events)
         self.event_listener_config = event_listener_config
+        self.cached_integration: dict[str, Any] | None = None
+
+    async def get_current_integration_cached(self) -> dict[str, Any]:
+        if self.cached_integration:
+            return self.cached_integration
+
+        self.cached_integration = await ocean.port_client.get_current_integration()
+        return self.cached_integration
+
+    async def get_saas_resync_initialization_and_interval(
+        self,
+    ) -> tuple[int | None, datetime.datetime | None]:
+        """
+        Get the scheduled resync interval and the last updated time of the integration config for the saas application.
+        interval is the saas configured resync interval time.
+        start_time is the last updated time of the integration config.
+        return: (interval, start_time)
+        """
+        if not ocean.app.is_saas():
+            return (None, None)
+
+        try:
+            integration = await self.get_current_integration_cached()
+        except Exception as e:
+            logger.exception(f"Error occurred while getting current integration {e}")
+            return (None, None)
+
+        interval_str = (
+            integration.get("spec", {})
+            .get("appSpec", {})
+            .get("scheduledResyncInterval")
+        )
+
+        if not interval_str:
+            logger.error(
+                "Unexpected: scheduledResyncInterval not found for Saas integration, Cannot predict the next resync"
+            )
+            return (None, None)
+
+        last_updated_saas_integration_config_str = integration.get(
+            "statusInfo", {}
+        ).get("updatedAt")
+
+        # we use the last updated time of the integration config as the start time since in saas application the interval is configured by the user from the portal
+        if not last_updated_saas_integration_config_str:
+            logger.error(
+                "Unexpected: updatedAt not found for Saas integration, Cannot predict the next resync"
+            )
+            return (None, None)
+
+        return (
+            convert_to_minutes(interval_str),
+            convert_str_to_utc_datetime(last_updated_saas_integration_config_str),
+        )
+
+    async def _before_resync(self) -> None:
+        if not ocean.app.is_saas():
+            # in case of non-saas, we still want to update the state before and after the resync
+            await super()._before_resync()
+            return
+
+        (interval, start_time) = (
+            await self.get_saas_resync_initialization_and_interval()
+        )
+        await ocean.app.resync_state_updater.update_before_resync(interval, start_time)
+
+    async def _after_resync(self) -> None:
+        if not ocean.app.is_saas():
+            # in case of non-saas, we still want to update the state before and after the resync
+            await super()._after_resync()
+            return
+
+        (interval, start_time) = (
+            await self.get_saas_resync_initialization_and_interval()
+        )
+        await ocean.app.resync_state_updater.update_after_resync(
+            IntegrationStateStatus.Completed, interval, start_time
+        )
+
+    async def _on_resync_failure(self, e: Exception) -> None:
+        if not ocean.app.is_saas():
+            # in case of non-saas, we still want to update the state before and after the resync
+            await super()._after_resync()
+            return
+
+        (interval, start_time) = (
+            await self.get_saas_resync_initialization_and_interval()
+        )
+        await ocean.app.resync_state_updater.update_after_resync(
+            IntegrationStateStatus.Failed, interval, start_time
+        )
 
     async def _start(self) -> None:
         """
@@ -53,7 +145,7 @@ class OnceEventListener(BaseEventListener):
         async def resync_and_exit() -> None:
             logger.info("Once event listener started")
             try:
-                await self.events["on_resync"]({})
+                await self._resync({})
             except Exception:
                 # we catch all exceptions here to make sure the application will exit gracefully
                 logger.exception("Error occurred while resyncing")