port-ocean 0.5.17__tar.gz → 0.5.21__tar.gz

{port_ocean-0.5.17 → port_ocean-0.5.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.5.17
+Version: 0.5.21
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml

@@ -1,4 +1,3 @@
-type: {{cookiecutter.integration_slug}}
 description: {{cookiecutter.integration_name}} integration for Port Ocean
 icon: Cookiecutter # Should be one of the available icons in Port
 features:

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/pyproject.toml

@@ -1,5 +1,5 @@
 [tool.poetry]
-name = "{{cookiecutter.integration_name}}"
+name = "{{cookiecutter.integration_slug}}"
 version = "0.1.0"
 description = "{{cookiecutter.integration_short_description}}"
 authors = ["{{cookiecutter.full_name}} <{{cookiecutter.email}}>"]

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/clients/port/mixins/integrations.py

@@ -6,6 +6,7 @@ from starlette import status
 
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.utils import handle_status_code
+from port_ocean.log.sensetive import sensitive_log_filter
 
 if TYPE_CHECKING:
     from port_ocean.core.handlers.port_app_config.models import PortAppConfig
@@ -137,3 +138,18 @@ class IntegrationClientMixin:
         )
         handle_status_code(response)
         logger.debug("Logs successfully ingested")
+
+    async def ingest_integration_kind_examples(
+        self, kind: str, data: list[dict[str, Any]], should_log: bool = True
+    ):
+        logger.debug(f"Ingesting examples for kind: {kind}")
+        headers = await self.auth.headers()
+        response = await self.client.post(
+            f"{self.auth.api_url}/integration/{self.integration_identifier}/kinds/{kind}/examples",
+            headers=headers,
+            json={
+                "examples": sensitive_log_filter.mask_object(data, full_hide=True),
+            },
+        )
+        handle_status_code(response, should_log=should_log)
+        logger.debug(f"Examples for kind {kind} successfully ingested")

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/config/base.py

@@ -16,16 +16,15 @@ PROVIDER_CONFIG_PATTERN = r"^[a-zA-Z0-9]+ .*$"
 
 def read_yaml_config_settings_source(
     settings: "BaseOceanSettings", base_path: str
-) -> str:
+) -> dict[str, Any]:
     yaml_file = getattr(settings.__config__, "yaml_file", "")
 
     assert yaml_file, "Settings.yaml_file not properly configured"
     path = Path(base_path, yaml_file)
 
     if not path.exists():
-        raise FileNotFoundError(f"Could not open yaml settings file at: {path}")
-
-    return path.read_text("utf-8")
+        return {}
+    return yaml.safe_load(path.read_text("utf-8"))
 
 
 def parse_config_provider(value: str) -> tuple[str, str]:
@@ -122,8 +121,7 @@ def decamelize_config(
 def load_providers(
     settings: "BaseOceanSettings", existing_values: dict[str, Any], base_path: str
 ) -> dict[str, Any]:
-    yaml_content = read_yaml_config_settings_source(settings, base_path)
-    data = yaml.safe_load(yaml_content)
+    data = read_yaml_config_settings_source(settings, base_path)
     snake_case_config = decamelize_config(settings, data)
     return parse_providers(settings, snake_case_config, existing_values)
 

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/config/settings.py

@@ -1,12 +1,14 @@
 from typing import Any, Literal
 
-from pydantic import Extra, AnyHttpUrl, parse_obj_as, validator
+from pydantic import Extra, AnyHttpUrl, parse_obj_as
+from pydantic.class_validators import root_validator
 from pydantic.env_settings import InitSettingsSource, EnvSettingsSource, BaseSettings
 from pydantic.fields import Field
 from pydantic.main import BaseModel
 
 from port_ocean.config.base import BaseOceanSettings, BaseOceanModel
 from port_ocean.core.event_listener import EventListenerSettingsType
+from port_ocean.utils.misc import get_integration_name
 
 LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"]
 
@@ -36,22 +38,34 @@ class PortSettings(BaseOceanModel, extra=Extra.allow):
     client_id: str = Field(..., sensitive=True)
     client_secret: str = Field(..., sensitive=True)
     base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io")
+    port_app_config_cache_ttl: int = 60
 
 
 class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
-    identifier: str
-    type: str
+    identifier: str = Field(..., min_length=1)
+    type: str = Field(..., min_length=1)
     config: dict[str, Any] | BaseModel
 
-    @validator("identifier", "type")
-    def validate_lower(cls, v: str) -> str:
-        return v.lower()
+    @root_validator(pre=True)
+    def a(cls, values: dict[str, Any]) -> dict[str, Any]:
+        integ_type = values.get("type")
+
+        if not integ_type:
+            integ_type = get_integration_name()
+
+        values["type"] = integ_type.lower() if integ_type else None
+        values["identifier"] = values.get(
+            "identifier", f"my-{integ_type}-integration".lower()
+        )
+
+        return values
 
 
 class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     initialize_port_resources: bool = True
     scheduled_resync_interval: int | None = None
     client_timeout: int = 30
+    send_raw_data_examples: bool = True
     port: PortSettings
     event_listener: EventListenerSettingsType
     integration: IntegrationSettings

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/core/event_listener/base.py

@@ -1,5 +1,4 @@
 from abc import abstractmethod
-from asyncio import Task
 from typing import TypedDict, Callable, Any, Awaitable
 
 from pydantic import Extra
@@ -22,7 +21,6 @@ class BaseEventListener:
         events: EventListenerEvents,
     ):
         self.events = events
-        self._tasks_to_close: list[Task[Any]] = []
 
     async def start(self) -> None:
         signal_handler.register(self._stop)
@@ -32,11 +30,6 @@ class BaseEventListener:
     async def _start(self) -> None:
         pass
 
-    def stop(self) -> None:
-        self._stop()
-        for task in self._tasks_to_close:
-            task.cancel()
-
     def _stop(self) -> None:
         """
         Can be used for event listeners that need cleanup before exiting.

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/core/event_listener/polling.py

@@ -10,6 +10,7 @@ from port_ocean.core.event_listener.base import (
     EventListenerSettings,
 )
 from port_ocean.utils.repeat import repeat_every
+from port_ocean.utils.signal import signal_handler
 
 
 class PollingEventListenerSettings(EventListenerSettings):
@@ -79,7 +80,7 @@ class PollingEventListener(BaseEventListener):
                 running_task: Task[Any] = get_event_loop().create_task(
                     self.events["on_resync"]({})  # type: ignore
                 )
-                self._tasks_to_close.append(running_task)
+                signal_handler.register(running_task.cancel)
 
                 await running_task
 

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/core/handlers/entity_processor/base.py

@@ -40,6 +40,7 @@ class BaseEntityProcessor(BaseHandler):
         mapping: ResourceConfig,
         raw_data: list[RAW_ITEM],
         parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
     ) -> EntitySelectorDiff:
         pass
 
@@ -48,6 +49,7 @@ class BaseEntityProcessor(BaseHandler):
         mapping: ResourceConfig,
         raw_data: list[RAW_ITEM],
         parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
     ) -> EntitySelectorDiff:
         """Public method to parse raw entity data and map it to an EntityDiff.
 
@@ -55,9 +57,12 @@ class BaseEntityProcessor(BaseHandler):
             mapping (ResourceConfig): The configuration for entity mapping.
             raw_data (list[RawEntity]): The raw data to be parsed.
             parse_all (bool): Whether to parse all data or just data that passed the selector.
+            send_raw_data_examples_amount (bool): Whether to send example data to the integration service.
 
         Returns:
             EntityDiff: The parsed entity differences.
         """
         with logger.contextualize(kind=mapping.kind):
-            return await self._parse_items(mapping, raw_data, parse_all)
+            return await self._parse_items(
+                mapping, raw_data, parse_all, send_raw_data_examples_amount
+            )

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -1,11 +1,13 @@
 import asyncio
 import functools
+from dataclasses import dataclass, field
 from functools import lru_cache
-from typing import Any
-from loguru import logger
+from typing import Any, Optional
 
 import pyjq as jq  # type: ignore
+from loguru import logger
 
+from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.models import Entity
@@ -14,6 +16,19 @@ from port_ocean.core.ocean_types import (
     EntitySelectorDiff,
 )
 from port_ocean.exceptions.core import EntityProcessorException
+from port_ocean.utils.queue_utils import process_in_queue
+
+
+@dataclass
+class MappedEntity:
+    """Represents the entity after applying the mapping
+
+    This class holds the mapping entity along with the selector boolean value and optionally the raw data.
+    """
+
+    entity: dict[str, Any] = field(default_factory=dict)
+    did_entity_pass_selector: bool = False
+    raw_data: Optional[dict[str, Any]] = None
 
 
 class JQEntityProcessor(BaseEntityProcessor):
@@ -77,13 +92,17 @@ class JQEntityProcessor(BaseEntityProcessor):
         raw_entity_mappings: dict[str, Any],
         selector_query: str,
         parse_all: bool = False,
-    ) -> tuple[dict[str, Any], bool]:
+    ) -> MappedEntity:
         should_run = await self._search_as_bool(data, selector_query)
         if parse_all or should_run:
             mapped_entity = await self._search_as_object(data, raw_entity_mappings)
-            return mapped_entity, should_run
+            return MappedEntity(
+                mapped_entity,
+                did_entity_pass_selector=should_run,
+                raw_data=data if should_run else None,
+            )
 
-        return {}, False
+        return MappedEntity()
 
     async def _calculate_entity(
         self,
@@ -92,7 +111,7 @@ class JQEntityProcessor(BaseEntityProcessor):
         items_to_parse: str | None,
         selector_query: str,
         parse_all: bool = False,
-    ) -> list[tuple[dict[str, Any], bool]]:
+    ) -> list[MappedEntity]:
         if items_to_parse:
             items = await self._search(data, items_to_parse)
             if isinstance(items, list):
@@ -117,40 +136,58 @@ class JQEntityProcessor(BaseEntityProcessor):
                     data, raw_entity_mappings, selector_query, parse_all
                 )
             ]
-        return [({}, False)]
+        return [MappedEntity()]
+
+    @staticmethod
+    async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
+        try:
+            if data:
+                await ocean.port_client.ingest_integration_kind_examples(
+                    kind, data, should_log=False
+                )
+        except Exception as ex:
+            logger.warning(
+                f"Failed to send raw data example {ex}",
+                exc_info=True,
+            )
 
     async def _parse_items(
         self,
         mapping: ResourceConfig,
         raw_results: list[RAW_ITEM],
         parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
     ) -> EntitySelectorDiff:
         raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
             exclude_unset=True
         )
-        calculate_entities_tasks = [
-            asyncio.create_task(
-                self._calculate_entity(
-                    data,
-                    raw_entity_mappings,
-                    mapping.port.items_to_parse,
-                    mapping.selector.query,
-                    parse_all,
-                )
-            )
-            for data in raw_results
-        ]
-        calculate_entities_results = await asyncio.gather(*calculate_entities_tasks)
+
+        calculated_entities_results = await process_in_queue(
+            raw_results,
+            self._calculate_entity,
+            raw_entity_mappings,
+            mapping.port.items_to_parse,
+            mapping.selector.query,
+            parse_all,
+        )
 
         passed_entities = []
         failed_entities = []
-        for entities_results in calculate_entities_results:
-            for entity, did_entity_pass_selector in entities_results:
-                if entity.get("identifier") and entity.get("blueprint"):
-                    parsed_entity = Entity.parse_obj(entity)
-                    if did_entity_pass_selector:
+        examples_to_send: list[dict[str, Any]] = []
+        for entities_results in calculated_entities_results:
+            for result in entities_results:
+                if result.entity.get("identifier") and result.entity.get("blueprint"):
+                    parsed_entity = Entity.parse_obj(result.entity)
+                    if result.did_entity_pass_selector:
                         passed_entities.append(parsed_entity)
+                        if (
+                            len(examples_to_send) < send_raw_data_examples_amount
+                            and result.raw_data is not None
+                        ):
+                            examples_to_send.append(result.raw_data)
                     else:
                         failed_entities.append(parsed_entity)
 
+        await self._send_examples(examples_to_send, mapping.kind)
+
         return EntitySelectorDiff(passed=passed_entities, failed=failed_entities)

port_ocean-0.5.21/port_ocean/core/handlers/port_app_config/base.py

@@ -0,0 +1,82 @@
+from abc import abstractmethod
+from typing import Type, Any
+
+from loguru import logger
+from pydantic import ValidationError
+
+from port_ocean.context.event import event
+from port_ocean.context.ocean import PortOceanContext
+from port_ocean.core.handlers.base import BaseHandler
+from port_ocean.core.handlers.port_app_config.models import PortAppConfig
+from port_ocean.utils.misc import get_time
+
+
+class PortAppConfigCache:
+    _port_app_config: PortAppConfig | None
+    _retrieval_time: float
+
+    def __init__(self, cache_ttl: int):
+        self._cache_ttl = cache_ttl
+
+    @property
+    def port_app_config(self) -> PortAppConfig:
+        if self._port_app_config is None:
+            raise ValueError("Port app config is not set")
+        return self._port_app_config
+
+    @port_app_config.setter
+    def port_app_config(self, value: PortAppConfig) -> None:
+        self._retrieval_time = get_time()
+        self._port_app_config = value
+
+    @property
+    def is_cache_invalid(self) -> bool:
+        return (
+            not self._port_app_config
+            or self._retrieval_time + self._cache_ttl < get_time()
+        )
+
+
+class BasePortAppConfig(BaseHandler):
+    """Abstract base class for managing port application configurations.
+
+    This class defines methods for obtaining and processing port application configurations.
+
+    Attributes:
+        context (Any): The context to be used during port application configuration.
+        CONFIG_CLASS (Type[PortAppConfig]): The class used for defining port application configuration settings.
+    """
+
+    CONFIG_CLASS: Type[PortAppConfig] = PortAppConfig
+
+    def __init__(self, context: PortOceanContext):
+        super().__init__(context)
+        self._app_config_cache = PortAppConfigCache(
+            self.context.config.port.port_app_config_cache_ttl
+        )
+
+    @abstractmethod
+    async def _get_port_app_config(self) -> dict[str, Any]:
+        pass
+
+    async def get_port_app_config(self, use_cache: bool = True) -> PortAppConfig:
+        """
+        Retrieve and parse the port application configuration.
+
+        :param use_cache: Determines whether to use the cached port-app-config if it exists, or to fetch it regardless
+        :return: The parsed port application configuration.
+        """
+        if not use_cache or self._app_config_cache.is_cache_invalid:
+            raw_config = await self._get_port_app_config()
+            try:
+                self._app_config_cache.port_app_config = self.CONFIG_CLASS.parse_obj(
+                    raw_config
+                )
+            except ValidationError:
+                logger.error(
+                    "Invalid port app config found. Please check that the integration has been configured correctly."
+                )
+                raise
+
+        event.port_app_config = self._app_config_cache.port_app_config
+        return self._app_config_cache.port_app_config

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -30,6 +30,9 @@ from port_ocean.core.utils import zip_and_sum
 from port_ocean.exceptions.core import OceanAbortException
 
 
+SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
+
+
 class SyncRawMixin(HandlerMixin, EventsMixin):
     """Mixin class for synchronization of raw constructed entities.
 
@@ -124,10 +127,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         self,
         raw_diff: list[tuple[ResourceConfig, list[RAW_ITEM]]],
         parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
     ) -> list[EntitySelectorDiff]:
         return await asyncio.gather(
             *(
-                self.entity_processor.parse_items(mapping, results, parse_all)
+                self.entity_processor.parse_items(
+                    mapping, results, parse_all, send_raw_data_examples_amount
+                )
                 for mapping, results in raw_diff
             )
         )
@@ -138,8 +144,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         results: list[dict[Any, Any]],
         user_agent_type: UserAgentType,
         parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
     ) -> EntitySelectorDiff:
-        objects_diff = await self._calculate_raw([(resource, results)], parse_all)
+        objects_diff = await self._calculate_raw(
+            [(resource, results)], parse_all, send_raw_data_examples_amount
+        )
         await self.entities_state_applier.upsert(
             objects_diff[0].passed, user_agent_type
         )
@@ -171,19 +180,32 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             else:
                 async_generators.append(result)
 
+        send_raw_data_examples_amount = (
+            SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
+        )
         entities = (
             await self._register_resource_raw(
-                resource_config, raw_results, user_agent_type
+                resource_config,
+                raw_results,
+                user_agent_type,
+                send_raw_data_examples_amount=send_raw_data_examples_amount,
             )
         ).passed
 
         for generator in async_generators:
             try:
                 async for items in generator:
+                    if send_raw_data_examples_amount > 0:
+                        send_raw_data_examples_amount = max(
+                            0, send_raw_data_examples_amount - len(entities)
+                        )
                     entities.extend(
                         (
                             await self._register_resource_raw(
-                                resource_config, items, user_agent_type
+                                resource_config,
+                                items,
+                                user_agent_type,
+                                send_raw_data_examples_amount=send_raw_data_examples_amount,
                             )
                         ).passed
                     )
@@ -362,7 +384,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             EventType.RESYNC,
             trigger_type=trigger_type,
         ):
-            app_config = await self.port_app_config_handler.get_port_app_config()
+            # If a resync is triggered due to a mappings change, we want to make sure that we have the updated version
+            # rather than the old cache
+            app_config = await self.port_app_config_handler.get_port_app_config(
+                use_cache=False
+            )
 
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/core/utils.py

@@ -28,38 +28,34 @@ def is_same_entity(first_entity: Entity, second_entity: Entity) -> bool:
     )
 
 
-def get_unique(array: list[Entity]) -> list[Entity]:
-    result: list[Entity] = []
-    for item in array:
-        if all(not is_same_entity(item, seen_item) for seen_item in result):
-            result.append(item)
-    return result
-
-
 def get_port_diff(
     before: Iterable[Entity],
     after: Iterable[Entity],
 ) -> EntityPortDiff:
-    return EntityPortDiff(
-        deleted=get_unique(
-            [
-                item
-                for item in before
-                if not any(is_same_entity(item, item_after) for item_after in after)
-            ],
-        ),
-        created=get_unique(
-            [
-                item
-                for item in after
-                if not any(is_same_entity(item, item_before) for item_before in before)
-            ],
-        ),
-        modified=get_unique(
-            [
-                item
-                for item in after
-                if any(is_same_entity(item, item_before) for item_before in before)
-            ],
-        ),
-    )
+    before_dict = {}
+    after_dict = {}
+    created = []
+    modified = []
+    deleted = []
+
+    # Create dictionaries for before and after lists
+    for entity in before:
+        key = (entity.identifier, entity.blueprint)
+        before_dict[key] = entity
+
+    for entity in after:
+        key = (entity.identifier, entity.blueprint)
+        after_dict[key] = entity
+
+    # Find created, modified, and deleted objects
+    for key, obj in after_dict.items():
+        if key not in before_dict:
+            created.append(obj)
+        else:
+            modified.append(obj)
+
+    for key, obj in before_dict.items():
+        if key not in after_dict:
+            deleted.append(obj)
+
+    return EntityPortDiff(created=created, modified=modified, deleted=deleted)

{port_ocean-0.5.17 → port_ocean-0.5.21}/port_ocean/log/sensetive.py

@@ -1,5 +1,5 @@
 import re
-from typing import Callable, TYPE_CHECKING
+from typing import Any, Callable, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from loguru import Record
@@ -35,16 +35,31 @@ class SensitiveLogFilter:
             [re.compile(re.escape(token.strip())) for token in tokens if token.strip()]
         )
 
+    def mask_string(self, string: str, full_hide: bool = False) -> str:
+        masked_string = string
+        for pattern in self.compiled_patterns:
+            replace: Callable[[re.Match[str]], str] | str = (
+                "[REDACTED]"
+                if full_hide
+                else lambda match: match.group()[:6] + "[REDACTED]"
+            )
+            masked_string = pattern.sub(replace, masked_string)
+        return masked_string
+
+    def mask_object(self, obj: Any, full_hide: bool = False) -> Any:
+        if isinstance(obj, str):
+            return self.mask_string(obj, full_hide)
+        if isinstance(obj, list):
+            return [self.mask_object(o, full_hide) for o in obj]
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                obj[k] = self.mask_object(v, full_hide)
+
+        return obj
+
     def create_filter(self, full_hide: bool = False) -> Callable[["Record"], bool]:
         def _filter(record: "Record") -> bool:
-            for pattern in self.compiled_patterns:
-                replace: Callable[[re.Match[str]], str] | str = (
-                    "[REDACTED]"
-                    if full_hide
-                    else lambda match: match.group()[:6] + "[REDACTED]"
-                )
-                record["message"] = pattern.sub(replace, record["message"])
-
+            record["message"] = self.mask_string(record["message"], full_hide)
             return True
 
         return _filter