port-ocean 0.28.2__py3-none-any.whl → 0.29.0__py3-none-any.whl

port_ocean/core/handlers/entity_processor/jq_input_evaluator.py

@@ -0,0 +1,137 @@
+import re
+from enum import Enum
+
+# This file is used to classify the input that a jq expression to run on.
+# It is used to determine if the jq expression can be executed without providing any JSON input (const expressions)
+# or on a single item (in items to parse situation)
+# or on all the data
+
+
+class InputClassifyingResult(Enum):
+    NONE = 1
+    SINGLE = 2
+    ALL = 3
+
+
+# Functions/filters that (even without ".") still require/assume input
+_INPUT_DEPENDENT_FUNCS = r"""
+\b(
+    map|select|reverse|sort|sort_by|unique|unique_by|group_by|flatten|transpose|
+    split|explode|join|add|length|has|in|index|indices|contains|
+    paths|leaf_paths|keys|keys_unsorted|values|to_entries|with_entries|from_entries|
+    del|delpaths|walk|reduce|foreach|input|inputs|limit|first|last|nth|
+    while|until|recurse|recurse_down|bsearch|combinations|permutations
+)\b
+"""
+
+_INPUT_DEPENDENT_RE = re.compile(_INPUT_DEPENDENT_FUNCS, re.VERBOSE)
+
+
+# String literal handling (jq uses double quotes for strings)
+_STRING_LITERAL_RE = re.compile(r'"(?:\\.|[^"\\])*"')
+_STRING_ONLY_RE = re.compile(r'^\s*"(?:\\.|[^"\\])*"\s*$')
+_NUMBER_ONLY_RE = re.compile(r"^\s*-?\d+(\.\d+)?\s*$")
+
+
+def _mask_strings(expr: str) -> str:
+    """
+    Replace string literals with 'S' strings so '.' inside quotes don't count.
+    Example:
+        - '"this is a string"' ---> 'S'
+        - '"sting" + .field'. ---> 'S + .field'
+    """
+    return _STRING_LITERAL_RE.sub("S", expr)
+
+
+def _mask_numbers(expr: str) -> str:
+    """
+    Replace number literals with 'N' so decimal points in numbers don't count as input references.
+    Example:
+        - '3.14' ---> 'N'
+        - '42 + 3.14' ---> 'N + N'
+    """
+    # Pattern to match numbers (integers and decimals, with optional sign)
+    number_pattern = re.compile(r"[-+]?\d+(?:\.\d+)?")
+    return number_pattern.sub("N", expr)
+
+
+def can_expression_run_with_no_input(selector_query: str) -> bool:
+    """
+    Returns True if the jq expression can be executed without providing any JSON input.
+    Rules:
+      - Whitespace-only => No Input Required
+      - A pure string literal => No Input Required (even if it contains '.')
+      - After masking strings, if it contains '.' => Input Required
+      - Disallow known input-dependent functions (functions that require input)
+      - After masking strings, if it contains only operators and numbers and 'S' => No Input Required
+      - Allow null/true/false/number/range/empty, and array/object literals that
+        don't reference input (no '.' after masking strings) => No Input Required
+    Example:
+        - blueprint: '"newRelicService"' in mapping, selector_query param would be '"newRelicService"' => No Input Required
+    """
+    s = selector_query.strip()
+    if s == "":
+        return True  # whitespace-only
+
+    # Pure string literal is nullary
+    if _STRING_ONLY_RE.match(s):
+        return True
+
+    # First mask strings, then mask numbers to prevent decimal points in numbers from being treated as input references
+    masked = _mask_strings(s).strip()
+    masked = _mask_numbers(masked).strip()
+
+    # If it contains any known input-dependent functions, don't shortcut
+    if _INPUT_DEPENDENT_RE.search(masked):
+        return False
+
+    # If it contains only operators and 'S'/'N', it can be executed with no input
+    # Example:
+    # - '"abc" + "def"' ---> 'S + S' => No Input Required
+    # - '3.14 + 2.5' ---> 'N + N' => No Input Required
+    # if re.fullmatch(
+    #     r"(?:S|N)(?:\s*[+\-*/]\s*(?:S|N))*",
+    #     masked,
+    # ):
+    #     return True
+
+    if "." not in masked:
+        return True
+
+    return False
+
+
+def _can_expression_run_on_single_item(expr: str, key: str) -> bool:
+    """
+    Detect `.key` outside of quotes, as a standalone path segment beginning
+    after a non-word boundary (start, space, |, (, [, {, , or :) and not part
+    of `.something.key`.
+    assuming key = 'item'
+    Examples:
+        - .item.yaeli => true
+        - map(.item.yaeli) => true
+        - .body.item => false
+    """
+    if not key:
+        return False
+
+    masked = _mask_strings(expr)
+    masked = _mask_numbers(masked)
+    pattern = re.compile(rf"(?<![A-Za-z0-9_])\.{re.escape(key)}(?![A-Za-z0-9_])")
+    return bool(pattern.search(masked))
+
+
+def classify_input(
+    selector_query: str, single_item_key: str | None = None
+) -> InputClassifyingResult:
+    """
+    Returns the input evaluation result for the jq expression.
+    Conservative: requires NO '.' and must match a known nullary-safe pattern.
+    """
+    if can_expression_run_with_no_input(selector_query):
+        return InputClassifyingResult.NONE
+    if single_item_key and _can_expression_run_on_single_item(
+        selector_query, single_item_key
+    ):
+        return InputClassifyingResult.SINGLE
+    return InputClassifyingResult.ALL

port_ocean/core/handlers/port_app_config/models.py

@@ -29,7 +29,9 @@ class EntityMapping(BaseModel):
 
     @property
     def is_using_search_identifier(self) -> bool:
-        return isinstance(self.identifier, dict)
+        return isinstance(self.identifier, dict) or isinstance(
+            self.identifier, IngestSearchQuery
+        )
 
 
 class MappingsConfig(BaseModel):
@@ -39,7 +41,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
-    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
+    items_to_parse_name: str = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

port_ocean/core/handlers/resync_state_updater/updater.py

@@ -60,7 +60,9 @@ class ResyncStateUpdater:
     async def update_after_resync(
         self,
         status: Literal[
-            IntegrationStateStatus.Completed, IntegrationStateStatus.Failed
+            IntegrationStateStatus.Completed,
+            IntegrationStateStatus.Failed,
+            IntegrationStateStatus.Aborted,
         ] = IntegrationStateStatus.Completed,
         interval: int | None = None,
         custom_start_time: datetime.datetime | None = None,
@@ -100,4 +102,4 @@ class ResyncStateUpdater:
         await ocean.metrics.report_sync_metrics(
             kinds=[ocean.metrics.current_resource_kind()]
         )
-        ocean.metrics.event_id = ""
+        ocean.metrics.event_id = f"{ocean.metrics.event_id}-done"

port_ocean/core/handlers/webhook/abstract_webhook_processor.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from enum import StrEnum
 from loguru import logger
 
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
@@ -12,6 +13,17 @@ from .webhook_event import (
 )
 
 
+class WebhookProcessorType(StrEnum):
+    """Type of webhook processor"""
+
+    # For action-related webhooks
+    # (e.g. update finished action using the workflow runs webhook)
+    ACTION = "action"
+    # For regular webhooks
+    # (e.g. repository events that should be reflected as Entities in Port)
+    WEBHOOK = "webhook"
+
+
 class AbstractWebhookProcessor(ABC):
     """
     Abstract base class for webhook processors
@@ -47,6 +59,10 @@ class AbstractWebhookProcessor(ABC):
         self.event = event
         self.retry_count = 0
 
+    @classmethod
+    def get_processor_type(cls) -> WebhookProcessorType:
+        return WebhookProcessorType.WEBHOOK
+
     async def on_error(self, error: Exception) -> None:
         """Hook to handle errors during processing. Override if needed"""
         delay = self.calculate_retry_delay()

port_ocean/core/handlers/webhook/processor_manager.py

@@ -11,11 +11,17 @@ from port_ocean.core.handlers.queue.abstract_queue import AbstractQueue
 from port_ocean.core.integrations.mixins.events import EventsMixin
 from port_ocean.core.integrations.mixins.live_events import LiveEventsMixin
 from port_ocean.exceptions.webhook_processor import WebhookEventNotSupportedError
-from .webhook_event import WebhookEvent, WebhookEventRawResults, LiveEventTimestamp
+from port_ocean.core.handlers.webhook.webhook_event import (
+    WebhookEvent,
+    WebhookEventRawResults,
+    LiveEventTimestamp,
+)
 from port_ocean.context.event import event
 
-
-from .abstract_webhook_processor import AbstractWebhookProcessor
+from port_ocean.core.handlers.webhook.abstract_webhook_processor import (
+    AbstractWebhookProcessor,
+    WebhookProcessorType,
+)
 from port_ocean.utils.signal import SignalHandler
 from port_ocean.core.handlers.queue import LocalQueue
 
@@ -56,7 +62,7 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         while True:
             event = None
             matching_processors: List[
-                Tuple[ResourceConfig, AbstractWebhookProcessor]
+                Tuple[ResourceConfig | None, AbstractWebhookProcessor]
             ] = []
             try:
                 event = await queue.get()
@@ -133,16 +139,22 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
 
     async def _extract_matching_processors(
         self, webhook_event: WebhookEvent, path: str
-    ) -> list[tuple[ResourceConfig, AbstractWebhookProcessor]]:
+    ) -> list[tuple[ResourceConfig | None, AbstractWebhookProcessor]]:
         """Find and extract the matching processor for an event"""
 
-        created_processors: list[tuple[ResourceConfig, AbstractWebhookProcessor]] = []
+        created_processors: list[
+            tuple[ResourceConfig | None, AbstractWebhookProcessor]
+        ] = []
         event_processor_names = []
 
         for processor_class in self._processors_classes[path]:
             processor = processor_class(webhook_event.clone())
             if await processor.should_process_event(webhook_event):
                 event_processor_names.append(processor.__class__.__name__)
+                if processor.get_processor_type() == WebhookProcessorType.ACTION:
+                    created_processors.append((None, processor))
+                    continue
+
                 kinds = await processor.get_matching_kinds(webhook_event)
                 for kind in kinds:
                     for resource in event.port_app_config.resources:
@@ -179,7 +191,10 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         event.set_timestamp(LiveEventTimestamp.FinishedProcessingWithError)
 
     async def _process_single_event(
-        self, processor: AbstractWebhookProcessor, path: str, resource: ResourceConfig
+        self,
+        processor: AbstractWebhookProcessor,
+        path: str,
+        resource: ResourceConfig | None,
     ) -> WebhookEventRawResults:
         """Process a single event with a specific processor"""
         try:
@@ -199,7 +214,7 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             raise
 
     async def _execute_processor(
-        self, processor: AbstractWebhookProcessor, resource: ResourceConfig
+        self, processor: AbstractWebhookProcessor, resource: ResourceConfig | None
     ) -> WebhookEventRawResults:
         """Execute a single processor within a max processing time"""
         try:
@@ -213,7 +228,7 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             )
 
     async def _process_webhook_request(
-        self, processor: AbstractWebhookProcessor, resource: ResourceConfig
+        self, processor: AbstractWebhookProcessor, resource: ResourceConfig | None
     ) -> WebhookEventRawResults:
         """Process a webhook request with retry logic
 
@@ -235,9 +250,10 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         while True:
             try:
                 webhook_event_raw_results = await processor.handle_event(
-                    payload, resource
+                    payload, resource  # type: ignore[arg-type]
                 )
-                webhook_event_raw_results.resource = resource
+                if resource is not None:
+                    webhook_event_raw_results.resource = resource
                 break
 
             except Exception as e:
@@ -258,7 +274,9 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         return webhook_event_raw_results
 
     def register_processor(
-        self, path: str, processor: Type[AbstractWebhookProcessor]
+        self,
+        path: str,
+        processor: Type[AbstractWebhookProcessor],
     ) -> None:
         """Register a webhook processor for a specific path with optional filter
 

port_ocean/core/integrations/mixins/sync_raw.py

@@ -1,4 +1,5 @@
 import asyncio
+import sys
 import uuid
 from graphlib import CycleError
 import inspect
@@ -23,7 +24,7 @@ from port_ocean.core.integrations.mixins.utils import (
     resync_generator_wrapper,
     resync_function_wrapper,
 )
-from port_ocean.core.models import Entity, ProcessExecutionMode
+from port_ocean.core.models import Entity, IntegrationFeatureFlag, ProcessExecutionMode
 from port_ocean.core.ocean_types import (
     RAW_RESULT,
     RESYNC_RESULT,
@@ -116,13 +117,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
+                results.append(resync_generator_wrapper(task, resource_config.kind, resource_config.port.items_to_parse_name, resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
                 )
                 task = typing.cast(Callable[[str], Awaitable[RAW_RESULT]], task)
-                tasks.append(resync_function_wrapper(task, resource_config.kind))
+                tasks.append(resync_function_wrapper(task, resource_config.kind, resource_config.port.items_to_parse))
 
         logger.info(
             f"Found {len(tasks) + len(results)} resync tasks for {resource_config.kind}"
@@ -477,7 +478,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             bool: True if lakehouse data is enabled, False otherwise
         """
         flags = await ocean.port_client.get_organization_feature_flags()
-        if "LAKEHOUSE_ELIGIBLE" in flags and ocean.config.lakehouse_enabled:
+        if IntegrationFeatureFlag.LAKEHOUSE_ELIGIBLE in flags and ocean.config.lakehouse_enabled:
             return True
         return False
 
@@ -1002,7 +1003,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
-            multiprocessing.set_start_method("fork", True)
+            if sys.platform.startswith("win"):
+                # fork is not supported on windows
+                multiprocessing.set_start_method("spawn", True)
+            else:
+                multiprocessing.set_start_method("fork", True)
             try:
                 for index, resource in enumerate(app_config.resources):
                     logger.info(