port-ocean 0.28.2__py3-none-any.whl → 0.29.0__py3-none-any.whl

integrations/_infra/Dockerfile.Deb

@@ -48,6 +48,7 @@ RUN apt-get update \
         curl \
         acl \
         sudo \
+        jq \
     && apt-get clean
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}
@@ -69,18 +70,22 @@ COPY --from=base /app/.venv /app/.venv
 COPY ./integrations/_infra/init.sh /app/init.sh
 
 USER root
+
 # Ensure that ocean is available for all in path
 RUN chmod a+x /app/.venv/bin/ocean
 
 RUN chmod a+x /app/init.sh
 RUN ln -s /app/.venv/bin/ocean /usr/bin/ocean
+
 # Add ocean user to ssl certs group
-RUN setfacl -m u:ocean:rwX /etc/ssl/certs
+RUN setfacl -R -m u:ocean:rwX /etc/ssl/certs \
+    && setfacl -d -m u:ocean:rwX /etc/ssl/certs
 
 # Allow ocean user to run update-ca-certificates without password (secure, limited sudo)
 RUN echo "ocean ALL=(root) NOPASSWD: /usr/sbin/update-ca-certificates" >> /etc/sudoers.d/ocean-certs \
     && chmod 440 /etc/sudoers.d/ocean-certs
 
 USER ocean
+
 # Run the application
 CMD ["bash", "/app/init.sh"]

integrations/_infra/Dockerfile.local

@@ -26,6 +26,7 @@ RUN apt-get update \
         python3-pip \
         python3-poetry \
         build-essential\
+        jq \
         git \
         python3-venv \
         acl \

port_ocean/clients/port/authentication.py

@@ -1,4 +1,5 @@
 import re
+import jwt
 from typing import Any
 
 import httpx
@@ -90,6 +91,24 @@ class PortAuthentication:
             )
         return self.last_token_object.full_token
 
+    async def is_machine_user(self) -> bool:
+        # Ensure self.last_token_object is populated
+        await self.token
+        if not self.last_token_object:
+            raise ValueError("No token found")
+
+        payload: dict[str, Any] = jwt.decode(
+            self.last_token_object.access_token, options={"verify_signature": False}
+        )
+        is_machine_user = payload.get("isMachine")
+        if is_machine_user is None:
+            logger.warning(
+                "Can not determine if the user is a machine user directly, checking for personal token usage instead"
+            )
+            return not payload.get("personalToken", True)
+
+        return is_machine_user
+
     @staticmethod
     def _is_personal_token(client_id: str) -> bool:
         email_regex = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"

port_ocean/clients/port/client.py

@@ -3,6 +3,7 @@ from typing import Any
 from loguru import logger
 
 from port_ocean.clients.port.authentication import PortAuthentication
+from port_ocean.clients.port.mixins.actions import ActionsClientMixin
 from port_ocean.clients.port.mixins.blueprints import BlueprintClientMixin
 from port_ocean.clients.port.mixins.entities import EntityClientMixin
 from port_ocean.clients.port.mixins.integrations import IntegrationClientMixin
@@ -24,6 +25,7 @@ class PortClient(
     BlueprintClientMixin,
     MigrationClientMixin,
     OrganizationClientMixin,
+    ActionsClientMixin,
 ):
     def __init__(
         self,
@@ -54,6 +56,7 @@ class PortClient(
         BlueprintClientMixin.__init__(self, self.auth, self.client)
         MigrationClientMixin.__init__(self, self.auth, self.client)
         OrganizationClientMixin.__init__(self, self.auth, self.client)
+        ActionsClientMixin.__init__(self, self.auth, self.client)
 
     async def get_kafka_creds(self) -> KafkaCreds:
         logger.info("Fetching organization kafka credentials")

port_ocean/clients/port/mixins/actions.py

@@ -0,0 +1,93 @@
+from typing import Any
+import httpx
+from loguru import logger
+from port_ocean.clients.port.authentication import PortAuthentication
+from port_ocean.clients.port.utils import handle_port_status_code
+from port_ocean.core.models import (
+    ActionRun,
+)
+from port_ocean.exceptions.execution_manager import RunAlreadyAcknowledgedError
+
+INTERNAL_ACTIONS_CLIENT_HEADER = {"x-port-automation-client": "true"}
+
+
+class ActionsClientMixin:
+    def __init__(self, auth: PortAuthentication, client: httpx.AsyncClient):
+        self.auth = auth
+        self.client = client
+
+    async def create_action(
+        self, action: dict[str, Any], should_log: bool = True
+    ) -> None:
+        logger.info(f"Creating action: {action}")
+        response = await self.client.post(
+            f"{self.auth.api_url}/actions",
+            json=action,
+            headers=await self.auth.headers(),
+        )
+
+        handle_port_status_code(response, should_log=should_log)
+
+    async def claim_pending_runs(
+        self, limit: int, visibility_timeout_ms: int
+    ) -> list[ActionRun]:
+        response = await self.client.post(
+            f"{self.auth.api_url}/actions/runs/claim-pending",
+            headers={**(await self.auth.headers()), **INTERNAL_ACTIONS_CLIENT_HEADER},
+            json={
+                "installationId": self.auth.integration_identifier,
+                "limit": limit,
+                "visibilityTimeoutMs": visibility_timeout_ms,
+            },
+        )
+        if response.is_error:
+            logger.error("Error claiming pending runs", error=response.text)
+            return []
+
+        return [ActionRun.parse_obj(run) for run in response.json().get("runs", [])]
+
+    async def get_run_by_external_id(self, external_id: str) -> ActionRun | None:
+        response = await self.client.get(
+            f"{self.auth.api_url}/actions/runs?version=v2&external_run_id={external_id}",
+            headers=await self.auth.headers(),
+        )
+        handle_port_status_code(response)
+        runs = response.json().get("runs", [])
+        return None if not len(runs) else ActionRun.parse_obj(runs[0])
+
+    async def patch_run(
+        self,
+        run_id: str,
+        run: ActionRun | dict[str, Any],
+        should_raise: bool = True,
+    ) -> None:
+        response = await self.client.patch(
+            f"{self.auth.api_url}/actions/runs/{run_id}",
+            headers=await self.auth.headers(),
+            json=run.dict() if isinstance(run, ActionRun) else run,
+        )
+        handle_port_status_code(response, should_raise=should_raise)
+
+    async def acknowledge_run(self, run_id: str) -> None:
+        try:
+            response = await self.client.patch(
+                f"{self.auth.api_url}/actions/runs/ack",
+                headers={
+                    **(await self.auth.headers()),
+                    **INTERNAL_ACTIONS_CLIENT_HEADER,
+                },
+                json={"runId": run_id},
+            )
+            handle_port_status_code(response)
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 409:
+                raise RunAlreadyAcknowledgedError()
+            raise
+
+    async def post_run_log(self, run_id: str, message: str) -> None:
+        response = await self.client.post(
+            f"{self.auth.api_url}/actions/runs/{run_id}/logs",
+            headers=await self.auth.headers(),
+            json={"message": message},
+        )
+        handle_port_status_code(response, should_raise=False)

port_ocean/clients/port/mixins/blueprints.py

@@ -81,18 +81,6 @@ class BlueprintClientMixin:
             handle_port_status_code(response, should_raise)
             return response.json().get("migrationId", "")
 
-    async def create_action(
-        self, action: dict[str, Any], should_log: bool = True
-    ) -> None:
-        logger.info(f"Creating action: {action}")
-        response = await self.client.post(
-            f"{self.auth.api_url}/actions",
-            json=action,
-            headers=await self.auth.headers(),
-        )
-
-        handle_port_status_code(response, should_log=should_log)
-
     async def create_scorecard(
         self,
         blueprint_identifier: str,

port_ocean/clients/port/mixins/entities.py

@@ -1,24 +1,24 @@
 import asyncio
+import json
 from typing import Any, Literal
 from urllib.parse import quote_plus
-import json
 
 import httpx
 from loguru import logger
-from port_ocean.context.ocean import ocean
+from starlette import status
+
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.types import RequestOptions, UserAgentType
 from port_ocean.clients.port.utils import (
-    handle_port_status_code,
     PORT_HTTP_MAX_CONNECTIONS_LIMIT,
+    handle_port_status_code,
 )
+from port_ocean.context.ocean import ocean
 from port_ocean.core.models import (
     BulkUpsertResponse,
     Entity,
     PortAPIErrorMessage,
 )
-from starlette import status
-
 from port_ocean.helpers.metric.metric import MetricPhase, MetricType
 
 ENTITIES_BULK_SAMPLES_SIZE = 10
@@ -484,54 +484,89 @@ class EntityClientMixin:
         parameters_to_include: list[str] | None = None,
     ) -> list[Entity]:
         if query is None:
-            datasource_prefix = f"port-ocean/{self.auth.integration_type}/"
-            datasource_suffix = (
-                f"/{self.auth.integration_identifier}/{user_agent_type.value}"
-            )
-            logger.info(
-                f"Searching entities with datasource prefix: {datasource_prefix} and suffix: {datasource_suffix}"
-            )
+            return await self._search_entities_by_datasource_paginated(user_agent_type)
+
+        return await self._search_entities_by_query(
+            user_agent_type=user_agent_type,
+            query=query,
+            parameters_to_include=parameters_to_include,
+        )
+
+    async def _search_entities_by_datasource_paginated(
+        self, user_agent_type: UserAgentType
+    ) -> list[Entity]:
+        datasource_prefix = f"port-ocean/{self.auth.integration_type}/"
+        datasource_suffix = (
+            f"/{self.auth.integration_identifier}/{user_agent_type.value}"
+        )
+        logger.info(
+            f"Searching entities with datasource prefix: {datasource_prefix} and suffix: {datasource_suffix}"
+        )
+
+        next_from: str | None = None
+        aggregated_entities: list[Entity] = []
+        while True:
+            request_body: dict[str, Any] = {
+                "datasource_prefix": datasource_prefix,
+                "datasource_suffix": datasource_suffix,
+            }
+
+            if next_from:
+                request_body["from"] = next_from
 
             response = await self.client.post(
                 f"{self.auth.api_url}/blueprints/entities/datasource-entities",
-                json={
-                    "datasource_prefix": datasource_prefix,
-                    "datasource_suffix": datasource_suffix,
-                },
+                json=request_body,
                 headers=await self.auth.headers(user_agent_type),
                 extensions={"retryable": True},
             )
-        else:
-            default_query = {
-                "combinator": "and",
-                "rules": [
-                    {
-                        "property": "$datasource",
-                        "operator": "contains",
-                        "value": f"port-ocean/{self.auth.integration_type}/",
-                    },
-                    {
-                        "property": "$datasource",
-                        "operator": "contains",
-                        "value": f"/{self.auth.integration_identifier}/{user_agent_type.value}",
-                    },
-                ],
-            }
+            handle_port_status_code(response)
+            response_json = response.json()
+            aggregated_entities.extend(
+                Entity.parse_obj(result) for result in response_json.get("entities", [])
+            )
+            next_from = response_json.get("next")
+            if not next_from:
+                break
 
-            if query.get("rules"):
-                query["rules"].extend(default_query["rules"])
+        return aggregated_entities
 
-            logger.info(f"Searching entities with custom query: {query}")
-            response = await self.client.post(
-                f"{self.auth.api_url}/entities/search",
-                json=query,
-                headers=await self.auth.headers(user_agent_type),
-                params={
-                    "exclude_calculated_properties": "true",
-                    "include": parameters_to_include or ["blueprint", "identifier"],
+    async def _search_entities_by_query(
+        self,
+        user_agent_type: UserAgentType,
+        query: dict[Any, Any],
+        parameters_to_include: list[str] | None,
+    ) -> list[Entity]:
+        default_query = {
+            "combinator": "and",
+            "rules": [
+                {
+                    "property": "$datasource",
+                    "operator": "contains",
+                    "value": f"port-ocean/{self.auth.integration_type}/",
                 },
-                extensions={"retryable": True},
-            )
+                {
+                    "property": "$datasource",
+                    "operator": "contains",
+                    "value": f"/{self.auth.integration_identifier}/{user_agent_type.value}",
+                },
+            ],
+        }
+
+        if query.get("rules"):
+            query["rules"].extend(default_query["rules"])
+
+        logger.info(f"Searching entities with custom query: {query}")
+        response = await self.client.post(
+            f"{self.auth.api_url}/entities/search",
+            json=query,
+            headers=await self.auth.headers(user_agent_type),
+            params={
+                "exclude_calculated_properties": "true",
+                "include": parameters_to_include or ["blueprint", "identifier"],
+            },
+            extensions={"retryable": True},
+        )
 
         handle_port_status_code(response)
         return [Entity.parse_obj(result) for result in response.json()["entities"]]

port_ocean/clients/port/mixins/integrations.py

@@ -1,4 +1,5 @@
 import asyncio
+from datetime import datetime
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict
 from urllib.parse import quote_plus
 
@@ -13,8 +14,6 @@ from port_ocean.log.sensetive import sensitive_log_filter
 if TYPE_CHECKING:
     from port_ocean.core.handlers.port_app_config.models import PortAppConfig
 
-
-ORG_USE_PROVISIONED_DEFAULTS_FEATURE_FLAG = "USE_PROVISIONED_DEFAULTS"
 INTEGRATION_POLLING_INTERVAL_INITIAL_SECONDS = 3
 INTEGRATION_POLLING_INTERVAL_BACKOFF_FACTOR = 1.55
 INTEGRATION_POLLING_RETRY_LIMIT = 30
@@ -152,6 +151,7 @@ class IntegrationClientMixin:
         changelog_destination: dict[str, Any],
         port_app_config: Optional["PortAppConfig"] = None,
         create_port_resources_origin_in_port: Optional[bool] = False,
+        actions_processing_enabled: Optional[bool] = False,
     ) -> Dict[str, Any]:
         logger.info(f"Creating integration with id: {self.integration_identifier}")
         headers = await self.auth.headers()
@@ -161,6 +161,7 @@ class IntegrationClientMixin:
             "version": self.integration_version,
             "changelogDestination": changelog_destination,
             "config": {},
+            "actionsProcessingEnabled": actions_processing_enabled,
         }
 
         query_params = {}
@@ -189,6 +190,7 @@ class IntegrationClientMixin:
         _type: str | None = None,
         changelog_destination: dict[str, Any] | None = None,
         port_app_config: Optional["PortAppConfig"] = None,
+        actions_processing_enabled: Optional[bool] = False,
     ) -> dict:
         logger.info(f"Updating integration with id: {self.integration_identifier}")
         headers = await self.auth.headers()
@@ -199,6 +201,8 @@ class IntegrationClientMixin:
             json["changelogDestination"] = changelog_destination
         if port_app_config:
             json["config"] = port_app_config.to_request()
+
+        json["actionsProcessingEnabled"] = actions_processing_enabled
         json["version"] = self.integration_version
 
         response = await self.client.patch(
@@ -300,6 +304,7 @@ class IntegrationClientMixin:
             headers=headers,
             json={
                 "items": raw_data,
+                "extractionTimestamp": int(datetime.now().timestamp() * 1000),
             },
         )
         handle_port_status_code(response, should_log=False)

port_ocean/config/settings.py

@@ -1,5 +1,5 @@
 import platform
-from typing import Any, Literal, Optional, Type, cast
+from typing import Any, Literal, Optional, Type
 
 from pydantic import AnyHttpUrl, Extra, parse_obj_as, parse_raw_as
 from pydantic.class_validators import root_validator, validator
@@ -8,10 +8,14 @@ from pydantic.fields import Field
 from pydantic.main import BaseModel
 
 from port_ocean.config.base import BaseOceanModel, BaseOceanSettings
-from port_ocean.core.event_listener import EventListenerSettingsType
+from port_ocean.core.event_listener import (
+    EventListenerSettingsType,
+    PollingEventListenerSettings,
+)
 from port_ocean.core.models import (
     CachingStorageMode,
     CreatePortResourcesOrigin,
+    EventListenerType,
     ProcessExecutionMode,
     Runtime,
 )
@@ -80,6 +84,14 @@ class StreamingSettings(BaseOceanModel, extra=Extra.allow):
     location: str = Field(default="/tmp/ocean/streaming")
 
 
+class ActionsProcessorSettings(BaseOceanModel, extra=Extra.allow):
+    enabled: bool = Field(default=False)
+    runs_buffer_high_watermark: int = Field(default=100)
+    visibility_timeout_ms: int = Field(default=30000)
+    poll_check_interval_seconds: int = Field(default=10)
+    workers_count: int = Field(default=1)
+
+
 class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     _integration_config_model: BaseModel | None = None
 
@@ -94,7 +106,9 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     base_url: str | None = None
     port: PortSettings
     event_listener: EventListenerSettingsType = Field(
-        default=cast(EventListenerSettingsType, {"type": "POLLING"})
+        default_factory=lambda: PollingEventListenerSettings(
+            type=EventListenerType.POLLING
+        )
     )
     event_workers_count: int = 1
     # If an identifier or type is not provided, it will be generated based on the integration name
@@ -122,6 +136,9 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     yield_items_to_parse_batch_size: int = 10
 
     streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
+    actions_processor: ActionsProcessorSettings = Field(
+        default_factory=lambda: ActionsProcessorSettings()
+    )
 
     @validator("process_execution_mode")
     def validate_process_execution_mode(
@@ -197,3 +214,18 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
                 raise ValueError("This integration can't be ran as Saas")
 
         return runtime
+
+    @validator("actions_processor")
+    def validate_actions_processor(
+        cls, actions_processor: ActionsProcessorSettings
+    ) -> ActionsProcessorSettings:
+        if not actions_processor.enabled:
+            return actions_processor
+
+        spec = get_spec_file()
+        if not (spec and spec.get("actionsProcessingEnabled", False)):
+            raise ValueError(
+                "Serving as an actions processor is not currently supported for this integration."
+            )
+
+        return actions_processor

port_ocean/context/ocean.py

@@ -1,4 +1,4 @@
-from typing import Callable, TYPE_CHECKING, Any, Literal, Union
+from typing import Callable, TYPE_CHECKING, Any, Union
 
 from fastapi import APIRouter
 from port_ocean.helpers.metric.metric import Metrics
@@ -7,7 +7,7 @@ from werkzeug.local import LocalProxy
 
 from port_ocean.clients.port.types import UserAgentType
 
-from port_ocean.core.models import Entity
+from port_ocean.core.models import Entity, EventListenerType
 from port_ocean.core.ocean_types import (
     RESYNC_EVENT_LISTENER,
     START_EVENT_LISTENER,
@@ -26,6 +26,7 @@ if TYPE_CHECKING:
     from port_ocean.core.integrations.base import BaseIntegration
     from port_ocean.ocean import Ocean
     from port_ocean.clients.port.client import PortClient
+    from port_ocean.core.handlers.actions.abstract_executor import AbstractExecutor
 
 from loguru import logger
 
@@ -73,9 +74,7 @@ class PortOceanContext:
         return self.app.port_client
 
     @property
-    def event_listener_type(
-        self,
-    ) -> Literal["WEBHOOK", "KAFKA", "POLLING", "ONCE", "WEBHOOKS_ONLY"]:
+    def event_listener_type(self) -> EventListenerType:
         return self.app.config.event_listener.type
 
     def on_resync(
@@ -213,6 +212,9 @@ class PortOceanContext:
         """
         self.app.webhook_manager.register_processor(path, processor)
 
+    def register_action_executor(self, executor: "AbstractExecutor") -> None:
+        self.app.execution_manager.register_executor(executor)
+
 
 _port_ocean: PortOceanContext = PortOceanContext(None)
 

port_ocean/core/defaults/initialize.py

@@ -13,14 +13,16 @@ from port_ocean.core.defaults.common import (
     get_port_integration_defaults,
 )
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
-from port_ocean.core.models import Blueprint, CreatePortResourcesOrigin
+from port_ocean.core.models import (
+    Blueprint,
+    CreatePortResourcesOrigin,
+    IntegrationFeatureFlag,
+)
 from port_ocean.core.utils.utils import gather_and_split_errors_from_results
 from port_ocean.exceptions.port_defaults import (
     AbortDefaultCreationError,
 )
 
-ORG_USE_PROVISIONED_DEFAULTS_FEATURE_FLAG = "USE_PROVISIONED_DEFAULTS"
-
 
 def deconstruct_blueprints_to_creation_steps(
     raw_blueprints: list[dict[str, Any]],
@@ -75,6 +77,7 @@ async def _initialize_required_integration_settings(
                 integration_config.integration.type,
                 integration_config.event_listener.get_changelog_destination_details(),
                 port_app_config=default_mapping,
+                actions_processing_enabled=integration_config.actions_processor.enabled,
                 create_port_resources_origin_in_port=integration_config.create_port_resources_origin
                 == CreatePortResourcesOrigin.Port,
             )
@@ -101,9 +104,13 @@ async def _initialize_required_integration_settings(
         integration.get("changelogDestination") != changelog_destination
         or integration.get("installationAppType") != integration_config.integration.type
         or integration.get("version") != port_client.integration_version
+        or integration.get("actionsProcessingEnabled")
+        != integration_config.actions_processor.enabled
     ):
         await port_client.patch_integration(
-            integration_config.integration.type, changelog_destination
+            _type=integration_config.integration.type,
+            changelog_destination=changelog_destination,
+            actions_processing_enabled=integration_config.actions_processor.enabled,
         )
 
 
@@ -219,7 +226,7 @@ async def _initialize_defaults(
         )
     )
 
-    has_provision_feature_flag = ORG_USE_PROVISIONED_DEFAULTS_FEATURE_FLAG in (
+    has_provision_feature_flag = IntegrationFeatureFlag.USE_PROVISIONED_DEFAULTS in (
         await port_client.get_organization_feature_flags()
     )
 

port_ocean/core/event_listener/__init__.py

@@ -20,6 +20,10 @@ from port_ocean.core.event_listener.webhooks_only import (
     WebhooksOnlyEventListener,
     WebhooksOnlyEventListenerSettings,
 )
+from port_ocean.core.event_listener.actions_only import (
+    ActionsOnlyEventListener,
+    ActionsOnlyEventListenerSettings,
+)
 
 
 EventListenerSettingsType = (
@@ -28,6 +32,7 @@ EventListenerSettingsType = (
     | PollingEventListenerSettings
     | OnceEventListenerSettings
     | WebhooksOnlyEventListenerSettings
+    | ActionsOnlyEventListenerSettings
 )
 
 __all__ = [
@@ -42,4 +47,6 @@ __all__ = [
     "OnceEventListenerSettings",
     "WebhooksOnlyEventListener",
     "WebhooksOnlyEventListenerSettings",
+    "ActionsOnlyEventListener",
+    "ActionsOnlyEventListenerSettings",
 ]

port_ocean/core/event_listener/actions_only.py

@@ -0,0 +1,42 @@
+from typing import Literal
+from loguru import logger
+
+from port_ocean.core.event_listener.base import (
+    BaseEventListener,
+    EventListenerEvents,
+    EventListenerSettings,
+)
+from port_ocean.core.models import EventListenerType
+
+
+class ActionsOnlyEventListenerSettings(EventListenerSettings):
+    """
+    This class inherits from `EventListenerSettings`, which provides a foundation for creating event listener settings.
+    """
+
+    type: Literal[EventListenerType.ACTIONS_ONLY]
+    should_resync: bool = False
+    should_process_webhooks: bool = False
+
+
+class ActionsOnlyEventListener(BaseEventListener):
+    """
+    No resync event listener.
+
+    It is used to handle events exclusively through actions without supporting resync events.
+
+    Parameters:
+        events (EventListenerEvents): A dictionary containing event types and their corresponding event handlers.
+        event_listener_config (ActionsOnlyEventListenerSettings): The event listener configuration settings.
+    """
+
+    def __init__(
+        self,
+        events: EventListenerEvents,
+        event_listener_config: ActionsOnlyEventListenerSettings,
+    ):
+        super().__init__(events)
+        self.event_listener_config = event_listener_config
+
+    async def _start(self) -> None:
+        logger.info("Starting Actions-only event listener")