port-ocean 0.5.6__py3-none-any.whl → 0.17.8__py3-none-any.whl

port_ocean/config/base.py

@@ -1,11 +1,11 @@
 import os
 import re
-from pathlib import Path
 from types import GenericAlias
 from typing import Any
 
 import yaml
 from humps import decamelize
+from pathlib import Path
 from pydantic import BaseSettings
 from pydantic.env_settings import EnvSettingsSource, InitSettingsSource
 from pydantic.main import ModelMetaclass, BaseModel
@@ -14,18 +14,21 @@ PROVIDER_WRAPPER_PATTERN = r"{{ from (.*) }}"
 PROVIDER_CONFIG_PATTERN = r"^[a-zA-Z0-9]+ .*$"
 
 
-def read_yaml_config_settings_source(
-    settings: "BaseOceanSettings", base_path: str
-) -> str:
-    yaml_file = getattr(settings.__config__, "yaml_file", "")
+def read_yaml_config_settings_source(settings: "BaseOceanSettings") -> dict[str, Any]:
+    yaml_file = getattr(settings.Config, "yaml_file", "")
 
     assert yaml_file, "Settings.yaml_file not properly configured"
-    path = Path(base_path, yaml_file)
+    path = Path(
+        getattr(
+            settings,
+            "_base_path",
+        ),
+        yaml_file,
+    )
 
     if not path.exists():
-        raise FileNotFoundError(f"Could not open yaml settings file at: {path}")
-
-    return path.read_text("utf-8")
+        return {}
+    return yaml.safe_load(path.read_text("utf-8"))
 
 
 def parse_config_provider(value: str) -> tuple[str, str]:
@@ -120,16 +123,15 @@ def decamelize_config(
 
 
 def load_providers(
-    settings: "BaseOceanSettings", existing_values: dict[str, Any], base_path: str
+    settings: "BaseOceanSettings", existing_values: dict[str, Any]
 ) -> dict[str, Any]:
-    yaml_content = read_yaml_config_settings_source(settings, base_path)
-    data = yaml.safe_load(yaml_content)
+    data = read_yaml_config_settings_source(settings)
     snake_case_config = decamelize_config(settings, data)
     return parse_providers(settings, snake_case_config, existing_values)
 
 
 class BaseOceanSettings(BaseSettings):
-    base_path: str
+    _base_path: str = "./"
 
     def get_sensitive_fields_data(self) -> set[str]:
         return _get_sensitive_information(self)
@@ -152,9 +154,7 @@ class BaseOceanSettings(BaseSettings):
             return (
                 init_settings,
                 env_settings,
-                lambda s: load_providers(
-                    s, env_settings(s), init_settings.init_kwargs["base_path"]
-                ),
+                lambda s: load_providers(s, {**env_settings(s), **init_settings(s)}),
             )
 
 

port_ocean/config/settings.py

@@ -1,16 +1,20 @@
-from typing import Any, Literal
+from typing import Any, Literal, Type, cast
 
-from pydantic import Extra, AnyHttpUrl, parse_obj_as, validator
+from pydantic import Extra, AnyHttpUrl, parse_obj_as, parse_raw_as
+from pydantic.class_validators import root_validator, validator
+from pydantic.env_settings import InitSettingsSource, EnvSettingsSource, BaseSettings
 from pydantic.fields import Field
 from pydantic.main import BaseModel
 
 from port_ocean.config.base import BaseOceanSettings, BaseOceanModel
 from port_ocean.core.event_listener import EventListenerSettingsType
+from port_ocean.core.models import Runtime
+from port_ocean.utils.misc import get_integration_name, get_spec_file
 
 LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"]
 
 
-class ApplicationSettings(BaseOceanModel):
+class ApplicationSettings(BaseSettings):
     log_level: LogLevelType = "INFO"
     enable_http_logging: bool = True
     port: int = 8000
@@ -21,7 +25,13 @@ class ApplicationSettings(BaseOceanModel):
         env_file_encoding = "utf-8"
 
         @classmethod
-        def customise_sources(cls, init_settings, env_settings, *_, **__):  # type: ignore
+        def customise_sources(  # type: ignore
+            cls,
+            init_settings: InitSettingsSource,
+            env_settings: EnvSettingsSource,
+            *_,
+            **__,
+        ):
             return env_settings, init_settings
 
 
@@ -29,22 +39,80 @@ class PortSettings(BaseOceanModel, extra=Extra.allow):
     client_id: str = Field(..., sensitive=True)
     client_secret: str = Field(..., sensitive=True)
     base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io")
+    port_app_config_cache_ttl: int = 60
 
 
 class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
     identifier: str
     type: str
-    config: dict[str, Any] | BaseModel
+    config: Any = Field(default_factory=dict)
 
-    @validator("identifier", "type")
-    def validate_lower(cls, v: str) -> str:
-        return v.lower()
+    @root_validator(pre=True)
+    def root_validator(cls, values: dict[str, Any]) -> dict[str, Any]:
+        integ_type = values.get("type")
+
+        if not integ_type:
+            integ_type = get_integration_name()
+
+        values["type"] = integ_type.lower() if integ_type else None
+        if not values.get("identifier"):
+            values["identifier"] = f"my-{integ_type}-integration".lower()
+
+        return values
 
 
 class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
+    _integration_config_model: BaseModel | None = None
+
+    allow_environment_variables_jq_access: bool = True
     initialize_port_resources: bool = True
     scheduled_resync_interval: int | None = None
-    client_timeout: int = 30
+    client_timeout: int = 60
+    send_raw_data_examples: bool = True
     port: PortSettings
-    event_listener: EventListenerSettingsType
-    integration: IntegrationSettings
+    event_listener: EventListenerSettingsType = Field(
+        default=cast(EventListenerSettingsType, {"type": "POLLING"})
+    )
+    # If an identifier or type is not provided, it will be generated based on the integration name
+    integration: IntegrationSettings = Field(
+        default_factory=lambda: IntegrationSettings(type="", identifier="")
+    )
+    runtime: Runtime = Runtime.OnPrem
+    resources_path: str = Field(default=".port/resources")
+
+    @root_validator()
+    def validate_integration_config(cls, values: dict[str, Any]) -> dict[str, Any]:
+        if not (config_model := values.get("_integration_config_model")):
+            return values
+
+        # Using the integration dynamic config model to parse the config
+        def parse_config(model: Type[BaseModel], config: Any) -> BaseModel:
+            # In some cases, the config is parsed as a string so we need to handle it
+            # Example: when the config is loaded from the environment variables and there is an object inside the config
+            if isinstance(config, str):
+                return parse_raw_as(model, config)
+            else:
+                return parse_obj_as(model, config)
+
+        integration_config = values["integration"]
+        integration_config.config = parse_config(
+            config_model, integration_config.config
+        )
+
+        return values
+
+    @validator("runtime")
+    def validate_runtime(cls, runtime: Runtime) -> Runtime:
+        if runtime.is_saas_runtime:
+            spec = get_spec_file()
+            if spec is None:
+                raise ValueError(
+                    "Could not determine whether it's safe to run "
+                    "the integration due to not found spec.yaml."
+                )
+
+            saas_config = spec.get("saas")
+            if saas_config and not saas_config["enabled"]:
+                raise ValueError("This integration can't be ran as Saas")
+
+        return runtime

port_ocean/context/event.py

@@ -14,6 +14,7 @@ from typing import (
 from uuid import uuid4
 
 from loguru import logger
+from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
 from pydispatch import dispatcher  # type: ignore
 from werkzeug.local import LocalStack, LocalProxy
 
@@ -24,6 +25,7 @@ from port_ocean.exceptions.context import (
 )
 from port_ocean.utils.misc import get_time
 
+
 if TYPE_CHECKING:
     from port_ocean.core.handlers.port_app_config.models import (
         ResourceConfig,
@@ -50,6 +52,9 @@ class EventContext:
     _parent_event: Optional["EventContext"] = None
     _event_id: str = field(default_factory=lambda: str(uuid4()))
     _on_abort_callbacks: list[AbortCallbackFunction] = field(default_factory=list)
+    entity_topological_sorter: EntityTopologicalSorter = field(
+        default_factory=EntityTopologicalSorter
+    )
 
     def on_abort(self, func: AbortCallbackFunction) -> None:
         self._on_abort_callbacks.append(func)
@@ -125,10 +130,17 @@ async def event_context(
     event_type: str,
     trigger_type: TriggerType = "manual",
     attributes: dict[str, Any] | None = None,
+    parent_override: EventContext | None = None,
 ) -> AsyncIterator[EventContext]:
-    attributes = attributes or {}
+    parent = parent_override or _event_context_stack.top
+    parent_attributes = parent.attributes if parent else {}
+    entity_topological_sorter = (
+        parent.entity_topological_sorter
+        if parent and parent.entity_topological_sorter
+        else EntityTopologicalSorter()
+    )
 
-    parent = _event_context_stack.top
+    attributes = {**parent_attributes, **(attributes or {})}
     new_event = EventContext(
         event_type,
         trigger_type=trigger_type,
@@ -136,6 +148,7 @@ async def event_context(
         _parent_event=parent,
         # inherit port app config from parent event, so it can be used in nested events
         _port_app_config=parent.port_app_config if parent else None,
+        entity_topological_sorter=entity_topological_sorter,
     )
     _event_context_stack.push(new_event)
 
@@ -156,9 +169,9 @@ async def event_context(
         event_kind=event.event_type,
         event_id=event.id,
         event_parent_id=event.parent_id,
-        event_resource_kind=event.resource_config.kind
-        if event.resource_config
-        else None,
+        event_resource_kind=(
+            event.resource_config.kind if event.resource_config else None
+        ),
     ):
         logger.info("Event started")
         try:

port_ocean/context/ocean.py

@@ -23,6 +23,8 @@ if TYPE_CHECKING:
     from port_ocean.ocean import Ocean
     from port_ocean.clients.port.client import PortClient
 
+from loguru import logger
+
 
 class PortOceanContext:
     def __init__(self, app: Union["Ocean", None]) -> None:
@@ -63,14 +65,23 @@ class PortOceanContext:
         return self.app.port_client
 
     @property
-    def event_listener_type(self) -> Literal["WEBHOOK", "KAFKA", "POLLING", "ONCE"]:
+    def event_listener_type(
+        self,
+    ) -> Literal["WEBHOOK", "KAFKA", "POLLING", "ONCE", "WEBHOOKS_ONLY"]:
         return self.app.config.event_listener.type
 
     def on_resync(
         self,
         kind: str | None = None,
-    ) -> Callable[[RESYNC_EVENT_LISTENER], RESYNC_EVENT_LISTENER]:
-        def wrapper(function: RESYNC_EVENT_LISTENER) -> RESYNC_EVENT_LISTENER:
+    ) -> Callable[[RESYNC_EVENT_LISTENER | None], RESYNC_EVENT_LISTENER | None]:
+        def wrapper(
+            function: RESYNC_EVENT_LISTENER | None,
+        ) -> RESYNC_EVENT_LISTENER | None:
+            if not self.app.config.event_listener.should_resync:
+                logger.debug(
+                    "Webhook only event listener is used, resync events are ignored"
+                )
+                return None
             return self.integration.on_resync(function, kind)
 
         return wrapper

port_ocean/core/defaults/clean.py

@@ -4,6 +4,7 @@ from typing import Type
 import httpx
 from loguru import logger
 
+from port_ocean.config.settings import IntegrationConfiguration
 from port_ocean.context.ocean import ocean
 from port_ocean.core.defaults.common import (
     get_port_integration_defaults,
@@ -14,12 +15,13 @@ from port_ocean.core.handlers.port_app_config.models import PortAppConfig
 
 def clean_defaults(
     config_class: Type[PortAppConfig],
+    integration_config: IntegrationConfiguration,
     force: bool,
     wait: bool,
 ) -> None:
     try:
         asyncio.new_event_loop().run_until_complete(
-            _clean_defaults(config_class, force, wait)
+            _clean_defaults(config_class, integration_config, force, wait)
         )
 
     except Exception as e:
@@ -27,13 +29,18 @@ def clean_defaults(
 
 
 async def _clean_defaults(
-    config_class: Type[PortAppConfig], force: bool, wait: bool
+    config_class: Type[PortAppConfig],
+    integration_config: IntegrationConfiguration,
+    force: bool,
+    wait: bool,
 ) -> None:
     port_client = ocean.port_client
     is_exists = await is_integration_exists(port_client)
     if not is_exists:
         return None
-    defaults = get_port_integration_defaults(config_class)
+    defaults = get_port_integration_defaults(
+        config_class, integration_config.resources_path
+    )
     if not defaults:
         return None
 

port_ocean/core/defaults/common.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from typing import Type, Any, TypedDict, Optional
 
 import httpx
+from loguru import logger
 import yaml
 from pydantic import BaseModel, Field
 from starlette import status
@@ -24,7 +25,7 @@ class Preset(TypedDict):
 
 class Defaults(BaseModel):
     blueprints: list[dict[str, Any]] = []
-    actions: list[Preset] = []
+    actions: list[dict[str, Any]] = []
     scorecards: list[Preset] = []
     pages: list[dict[str, Any]] = []
     port_app_config: Optional[PortAppConfig] = Field(
@@ -77,18 +78,33 @@ def deconstruct_blueprints_to_creation_steps(
     )
 
 
+def is_valid_dir(path: Path) -> bool:
+    return path.is_dir()
+
+
 def get_port_integration_defaults(
-    port_app_config_class: Type[PortAppConfig], base_path: Path = Path(".")
+    port_app_config_class: Type[PortAppConfig],
+    custom_defaults_dir: Optional[str] = None,
+    base_path: Path = Path("."),
 ) -> Defaults | None:
-    defaults_dir = base_path / ".port/resources"
-    if not defaults_dir.exists():
-        return None
-
-    if not defaults_dir.is_dir():
-        raise UnsupportedDefaultFileType(
-            f"Defaults directory is not a directory: {defaults_dir}"
+    fallback_dir = base_path / ".port/resources"
+
+    if custom_defaults_dir and is_valid_dir(base_path / custom_defaults_dir):
+        defaults_dir = base_path / custom_defaults_dir
+    elif is_valid_dir(fallback_dir):
+        logger.info(
+            f"Could not find custom defaults directory {custom_defaults_dir}, falling back to {fallback_dir}",
+            fallback_dir=fallback_dir,
+            custom_defaults_dir=custom_defaults_dir,
+        )
+        defaults_dir = fallback_dir
+    else:
+        logger.warning(
+            f"Could not find defaults directory {fallback_dir}, skipping defaults"
         )
+        return None
 
+    logger.info(f"Loading defaults from {defaults_dir}", defaults_dir=defaults_dir)
     default_jsons = {}
     allowed_file_names = [
         field_model.alias for _, field_model in Defaults.__fields__.items()