port-ocean 0.30.7__tar.gz → 0.31.0__tar.gz

{port_ocean-0.30.7 → port_ocean-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.30.7
+Version: 0.31.0
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.30.7 → port_ocean-0.31.0}/integrations/_infra/Dockerfile.Deb

@@ -48,7 +48,7 @@ RUN apt-get update \
         curl \
         acl \
         sudo \
-        jq \
+        libyaml-dev \
     && apt-get clean
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}

{port_ocean-0.30.7 → port_ocean-0.31.0}/integrations/_infra/Dockerfile.local

@@ -26,10 +26,10 @@ RUN apt-get update \
         python3-pip \
         python3-poetry \
         build-essential\
-        jq \
         git \
         python3-venv \
         acl \
+        libyaml-dev \
     && apt-get clean
 
 ARG BUILD_CONTEXT
@@ -51,7 +51,6 @@ RUN rm -rf .venv-docker ${BUILD_CONTEXT}/.venv-docker
 RUN python3 -m venv .venv-docker
 RUN python3 -m venv ${BUILD_CONTEXT}/.venv-docker
 
-
 WORKDIR /app/${BUILD_CONTEXT}
 
 WORKDIR /app
@@ -60,6 +59,7 @@ RUN chown -R ocean:appgroup /app/${BUILD_CONTEXT} && chmod -R 755 /app/${BUILD_C
 RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
 # Add ocean user to ssl certs group
 RUN setfacl -m u:ocean:rwX /etc/ssl/certs
+
 USER ocean
 
 ENTRYPOINT ["./integrations/_infra/entry_local.sh"]

{port_ocean-0.30.7 → port_ocean-0.31.0}/integrations/_infra/entry_local.sh

@@ -23,4 +23,5 @@ source .venv-docker/bin/activate
 python -m pip install -e ../../
 
 python -m pip install debugpy
-python -m debugpy --listen 0.0.0.0:5678 --wait-for-client debug.py
+# python -m debugpy --listen 0.0.0.0:5678 --wait-for-client debug.py
+make run

port_ocean-0.31.0/integrations/_infra/hosts

@@ -0,0 +1,4 @@
+192.168.65.254 localhost
+192.168.65.254 api.localhost
+192.168.65.254 gw.localhost
+192.168.65.254 ingest.localhost

{port_ocean-0.30.7 → port_ocean-0.31.0}/port_ocean/config/settings.py

@@ -132,8 +132,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     upsert_entities_batch_max_length: int = 20
     upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
     lakehouse_enabled: bool = False
-    yield_items_to_parse: bool = True
-    yield_items_to_parse_batch_size: int = 10
+    yield_items_to_parse_batch_size: int = 500
 
     streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
     actions_processor: ActionsProcessorSettings = Field(

port_ocean-0.31.0/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -0,0 +1,375 @@
+import asyncio
+import json
+import re
+from asyncio import Task
+from dataclasses import dataclass, field
+from functools import lru_cache
+from typing import Any, Optional
+
+import jq  # type: ignore
+from loguru import logger
+
+from port_ocean.context.ocean import ocean
+from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.core.models import Entity
+from port_ocean.core.ocean_types import (
+    RAW_ITEM,
+    CalculationResult,
+    EntitySelectorDiff,
+)
+from port_ocean.core.utils.utils import (
+    gather_and_split_errors_from_results,
+    zip_and_sum,
+)
+from port_ocean.exceptions.core import EntityProcessorException
+from port_ocean.utils.queue_utils import process_in_queue
+
+
+class ExampleStates:
+    __succeed: list[dict[str, Any]]
+    __errors: list[dict[str, Any]]
+    __max_size: int
+
+    def __init__(self, max_size: int = 0) -> None:
+        """
+        Store two sequences:
+          - succeed: items that succeeded
+          - errors:  items that failed
+        """
+        self.__succeed = []
+        self.__errors = []
+        self.__max_size = max_size
+
+    def add_example(self, succeed: bool, item: dict[str, Any]) -> None:
+        if succeed:
+            self.__succeed.append(item)
+        else:
+            self.__errors.append(item)
+
+    def __len__(self) -> int:
+        """
+        Total number of items (successes + errors).
+        """
+        return len(self.__succeed) + len(self.__errors)
+
+    def get_examples(self, number: int = 0) -> list[dict[str, Any]]:
+        """
+        Return a list of up to number items, taking successes first,
+        """
+        if number <= 0:
+            number = self.__max_size
+        # how many from succeed?
+        s_count = min(number, len(self.__succeed))
+        result = list(self.__succeed[:s_count])
+        # how many more from errors?
+        e_count = number - s_count
+        if e_count > 0:
+            result.extend(self.__errors[:e_count])
+        return result
+
+
+@dataclass
+class MappedEntity:
+    """Represents the entity after applying the mapping
+
+    This class holds the mapping entity along with the selector boolean value and optionally the raw data.
+    """
+
+    entity: dict[str, Any] = field(default_factory=dict)
+    did_entity_pass_selector: bool = False
+    raw_data: Optional[dict[str, Any] | tuple[dict[str, Any], str]] = None
+    misconfigurations: dict[str, str] = field(default_factory=dict)
+
+
+class JQEntityProcessor(BaseEntityProcessor):
+    """Processes and parses entities using JQ expressions.
+
+    This class extends the BaseEntityProcessor and provides methods for processing and
+    parsing entities based on PyJQ queries. It supports compiling and executing PyJQ patterns,
+    searching for data in dictionaries, and transforming data based on object mappings.
+    """
+
+    @staticmethod
+    def _format_filter(filter: str) -> str:
+        """
+        Convert single quotes to double quotes in JQ expressions.
+        Only replaces single quotes that are opening or closing string delimiters,
+        not single quotes that are part of string content.
+        """
+        # Escape single quotes only if they are opening or closing a string
+        # Pattern matches:
+        # - Single quote at start of string or after whitespace (opening quote)
+        # - Single quote before whitespace or end of string (closing quote)
+        # Uses negative lookahead/lookbehind to avoid replacing quotes inside strings
+        # \1 and \2 will be empty for the alternative that didn't match, so \1"\2 works for both cases
+        # This matches the TypeScript pattern: /(^|\s)'(?!\s|")|(?<!\s|")'(\s|$)/g
+        formatted_filter = re.sub(
+            r'(^|\s)\'(?!\s|")|(?<!\s|")\'(\s|$)', r'\1"\2', filter
+        )
+        return formatted_filter
+
+    @lru_cache
+    def _compile(self, pattern: str) -> Any:
+        # Convert single quotes to double quotes for JQ compatibility
+        pattern = self._format_filter(pattern)
+        if not ocean.config.allow_environment_variables_jq_access:
+            pattern = "def env: {}; {} as $ENV | " + pattern
+        return jq.compile(pattern)
+
+    @staticmethod
+    def _stop_iterator_handler(func: Any) -> Any:
+        """
+        Wrap the function to handle StopIteration exceptions.
+        Prevents StopIteration from stopping the thread and skipping further queue processing.
+        """
+
+        def inner() -> Any:
+            try:
+                return func()
+            except StopIteration:
+                return None
+
+        return inner
+
+    @staticmethod
+    def _notify_mapping_issues(
+        entity_misconfigurations: dict[str, str],
+        missing_required_fields: bool,
+        entity_mapping_fault_counter: int,
+    ) -> None:
+        if len(entity_misconfigurations) > 0:
+            logger.error(
+                f"Unable to find valid data for: {entity_misconfigurations} (null, missing, or misconfigured)"
+            )
+        if missing_required_fields:
+            logger.error(
+                f"{entity_mapping_fault_counter} transformations of batch failed due to empty, null or missing values"
+            )
+
+    async def _search(self, data: dict[str, Any], pattern: str) -> Any:
+        try:
+            compiled_pattern = self._compile(pattern)
+            func = compiled_pattern.input_value(data)
+            return func.first()
+        except Exception as exc:
+            logger.error(
+                f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
+            )
+            return None
+
+    async def _search_as_bool(self, data: dict[str, Any] | str, pattern: str) -> bool:
+
+        compiled_pattern = self._compile(pattern)
+
+        func = compiled_pattern.input_value(data)
+
+        value = func.first()
+        if isinstance(value, bool):
+            return value
+        raise EntityProcessorException(
+            f"Expected boolean value, got value:{value} of type: {type(value)} instead"
+        )
+
+    async def _search_as_object(
+        self,
+        data: dict[str, Any],
+        obj: dict[str, Any],
+        misconfigurations: dict[str, str] | None = None,
+    ) -> dict[str, Any | None]:
+        """
+        Identify and extract the relevant value for the chosen key and populate it into the entity
+        :param data: the property itself that holds the key and the value, it is being passed to the task and we get back a task item,
+            if the data is a dict, we will recursively call this function again.
+        :param obj: the key that we want its value to be mapped into our entity.
+        :param misconfigurations: due to the recursive nature of this function,
+            we aim to have a dict that represents all of the misconfigured properties and when used recursively,
+            we pass this reference to misfoncigured object to add the relevant misconfigured keys.
+        :return: Mapped object with found value.
+        """
+
+        search_tasks: dict[
+            str, Task[dict[str, Any | None]] | list[Task[dict[str, Any | None]]]
+        ] = {}
+        for key, value in obj.items():
+            if isinstance(value, list):
+                search_tasks[key] = [
+                    asyncio.create_task(
+                        self._search_as_object(data, obj, misconfigurations)
+                    )
+                    for obj in value
+                ]
+
+            elif isinstance(value, dict):
+                search_tasks[key] = asyncio.create_task(
+                    self._search_as_object(data, value, misconfigurations)
+                )
+            else:
+                search_tasks[key] = asyncio.create_task(self._search(data, value))
+
+        result: dict[str, Any | None] = {}
+        for key, task in search_tasks.items():
+            try:
+                if isinstance(task, list):
+                    result_list = []
+                    for task in task:
+                        task_result = await task
+                        if task_result is None and misconfigurations is not None:
+                            misconfigurations[key] = obj[key]
+                        result_list.append(task_result)
+                    result[key] = result_list
+                else:
+                    task_result = await task
+                    if task_result is None and misconfigurations is not None:
+                        misconfigurations[key] = obj[key]
+                    result[key] = task_result
+            except Exception:
+                result[key] = None
+        return result
+
+    async def _get_mapped_entity(
+        self,
+        data: dict[str, Any],
+        raw_entity_mappings: dict[str, Any],
+        selector_query: str,
+        parse_all: bool = False,
+    ) -> MappedEntity:
+        should_run = await self._search_as_bool(data, selector_query)
+        if parse_all or should_run:
+            misconfigurations: dict[str, str] = {}
+            mapped_entity = await self._search_as_object(
+                data, raw_entity_mappings, misconfigurations
+            )
+            return MappedEntity(
+                mapped_entity,
+                did_entity_pass_selector=should_run,
+                raw_data=data if should_run else None,
+                misconfigurations=misconfigurations,
+            )
+
+        return MappedEntity()
+
+    async def _calculate_entity(
+        self,
+        data: dict[str, Any],
+        raw_entity_mappings: dict[str, Any],
+        selector_query: str,
+        parse_all: bool = False,
+    ) -> tuple[list[MappedEntity], list[Exception]]:
+        raw_data = [data.copy()]
+
+        entities, errors = await gather_and_split_errors_from_results(
+            [
+                self._get_mapped_entity(
+                    raw,
+                    raw_entity_mappings,
+                    selector_query,
+                    parse_all,
+                )
+                for raw in raw_data
+            ]
+        )
+        if errors:
+            logger.error(
+                f"Failed to calculate entities with {len(errors)} errors. errors: {errors}"
+            )
+        return entities, errors
+
+    @staticmethod
+    async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
+        try:
+            if data:
+                await ocean.port_client.ingest_integration_kind_examples(
+                    kind, data, should_log=False
+                )
+        except Exception as ex:
+            logger.warning(
+                f"Failed to send raw data example {ex}",
+                exc_info=True,
+            )
+
+    async def _parse_items(
+        self,
+        mapping: ResourceConfig,
+        raw_results: list[RAW_ITEM],
+        parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
+    ) -> CalculationResult:
+        raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
+            exclude_unset=True
+        )
+        logger.info(f"Parsing {len(raw_results)} raw results into entities")
+        calculated_entities_results, errors = zip_and_sum(
+            await process_in_queue(
+                raw_results,
+                self._calculate_entity,
+                raw_entity_mappings,
+                mapping.selector.query,
+                parse_all,
+            )
+        )
+        logger.debug(
+            f"Finished parsing raw results into entities with {len(errors)} errors. errors: {errors}"
+        )
+
+        passed_entities = []
+        failed_entities = []
+        examples_to_send = ExampleStates(send_raw_data_examples_amount)
+        entity_misconfigurations: dict[str, str] = {}
+        missing_required_fields: bool = False
+        entity_mapping_fault_counter: int = 0
+        for result in calculated_entities_results:
+            if len(result.misconfigurations) > 0:
+                entity_misconfigurations |= result.misconfigurations
+
+            if (
+                len(examples_to_send) < send_raw_data_examples_amount
+                and result.raw_data is not None
+            ):
+                examples_to_send.add_example(
+                    result.did_entity_pass_selector,
+                    self._get_raw_data_for_example(
+                        result.raw_data, mapping.port.items_to_parse_name
+                    ),
+                )
+
+            if result.entity.get("identifier") and result.entity.get("blueprint"):
+                parsed_entity = Entity.parse_obj(result.entity)
+                if result.did_entity_pass_selector:
+                    passed_entities.append(parsed_entity)
+                else:
+                    failed_entities.append(parsed_entity)
+            else:
+                missing_required_fields = True
+                entity_mapping_fault_counter += 1
+
+        self._notify_mapping_issues(
+            entity_misconfigurations,
+            missing_required_fields,
+            entity_mapping_fault_counter,
+        )
+
+        await self._send_examples(examples_to_send.get_examples(), mapping.kind)
+
+        return CalculationResult(
+            EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
+            errors,
+            misconfigured_entity_keys=entity_misconfigurations,
+        )
+
+    def _get_raw_data_for_example(
+        self,
+        data: dict[str, Any] | tuple[dict[str, Any], str],
+        items_to_parse_name: str,
+    ) -> dict[str, Any]:
+        if isinstance(data, tuple):
+            raw_data = json.loads(data[1])
+            return {
+                **(
+                    data[0]
+                    if items_to_parse_name in data[0]
+                    else {items_to_parse_name: data[0]}
+                ),
+                **raw_data,
+            }
+        return data

port_ocean-0.31.0/port_ocean/core/integrations/mixins/utils.py

@@ -0,0 +1,198 @@
+import asyncio
+import multiprocessing
+import re
+from contextlib import contextmanager
+from typing import Any, AsyncGenerator, Awaitable, Callable, Generator
+
+import ijson
+from loguru import logger
+
+from port_ocean.clients.port.utils import _http_client as _port_http_client
+from port_ocean.context.ocean import ocean
+from port_ocean.core.ocean_types import (
+    ASYNC_GENERATOR_RESYNC_TYPE,
+    RAW_RESULT,
+    RESYNC_EVENT_LISTENER,
+    RESYNC_RESULT,
+)
+from port_ocean.core.utils.utils import validate_result
+from port_ocean.exceptions.core import (
+    RawObjectValidationException,
+    OceanAbortException,
+    KindNotImplementedException,
+)
+from port_ocean.helpers.metric.metric import MetricType, MetricPhase
+from port_ocean.utils.async_http import _http_client
+
+def extract_jq_deletion_path_revised(jq_expression: str) -> str | None:
+    """
+    Revised function to extract a simple path suitable for del() by analyzing pipe segments.
+    """
+    expr = jq_expression.strip()
+
+    # 1. Handle surrounding parentheses and extract the main chain
+    if expr.startswith('('):
+        match_paren = re.match(r'\((.*?)\)', expr, re.DOTALL)
+        if match_paren:
+            chain = match_paren.group(1).strip()
+        else:
+            return None
+    else:
+        chain = expr
+
+    # 2. Split the chain by the main pipe operator (excluding pipes inside quotes or brackets,
+    # but for simplicity here, we split naively and check segments)
+    segments = chain.split('|')
+
+    # 3. Analyze each segment for a simple path
+    for segment in segments:
+        segment = segment.strip()
+
+        # Ignore variable assignment segments like '. as $root'
+        if re.match(r'^\.\s+as\s+\$\w+', segment):
+            continue
+
+        # Ignore identity and variable access like '.' or '$items'
+        if segment == '.' or segment.startswith('$'):
+            continue
+
+        # Look for the first genuine path accessor (e.g., .key, .[index], .key.nested, .key[0])
+        # This regex looks for a starting dot and follows it with path components
+        # (alphanumeric keys or bracketed accessors, where brackets can follow words directly)
+        # Pattern: Start with .word or .[index], then optionally more:
+        #   - .word (dot followed by word)
+        #   - [index] (bracket directly after word, no dot)
+        #   - .[index] (dot followed by bracket)
+        path_match = re.match(r'(\.[\w]+|\.\[[^\]]+\])(\.[\w]+|\[[^\]]+\]|\.\[[^\]]+\])*', segment)
+
+        if path_match:
+            path = path_match.group(0).strip()
+
+            # If the path is immediately followed by a simple fallback (// value),
+            # we consider the path complete.
+            if re.search(r'\s*//\s*(\[\]|null|\.|\{.*?\})', segment):
+                return path
+
+            # If the path is just a path segment followed by nothing or the end of a complex
+            # expression (like .file.content.raw) we return it.
+            return path
+
+    # Default case: No suitable path found after checking all segments
+    return None
+
+@contextmanager
+def resync_error_handling() -> Generator[None, None, None]:
+    try:
+        yield
+    except RawObjectValidationException as error:
+        err_msg = f"Failed to validate raw data for returned data from resync function, error: {error}"
+        logger.exception(err_msg)
+        raise OceanAbortException(err_msg) from error
+    except StopAsyncIteration:
+        raise
+    except Exception as error:
+        err_msg = f"Failed to execute resync function, error: {error}"
+        logger.exception(err_msg)
+        raise OceanAbortException(err_msg) from error
+
+
+async def resync_function_wrapper(
+    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str, items_to_parse: str | None = None
+) -> RAW_RESULT:
+    with resync_error_handling():
+        results = await fn(kind)
+        return validate_result(results)
+
+async def handle_items_to_parse(result: RAW_RESULT, items_to_parse_name: str, items_to_parse: str | None = None) -> AsyncGenerator[list[dict[str, Any]], None]:
+    delete_target = extract_jq_deletion_path_revised(items_to_parse) or '.'
+    jq_expression = f". | del({delete_target})"
+    batch_size = ocean.config.yield_items_to_parse_batch_size
+
+    for item in result:
+        lean_item = await ocean.app.integration.entity_processor._search(item, jq_expression)
+        items_to_parse_data = await ocean.app.integration.entity_processor._search(item, items_to_parse)
+        if not isinstance(items_to_parse_data, list):
+            logger.warning(
+                f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items_to_parse_data)}."
+                f" Skipping..."
+            )
+            continue
+        batch = []
+        while len(items_to_parse_data) > 0:
+            if (len(batch) >= batch_size):
+                yield batch
+                batch = []
+            merged_item = {**lean_item}
+            merged_item[items_to_parse_name] = items_to_parse_data.pop(0)
+            batch.append(merged_item)
+        if len(batch) > 0:
+            yield batch
+
+async def resync_generator_wrapper(
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
+) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    generator = fn(kind)
+    errors = []
+    try:
+        while True:
+            try:
+                with resync_error_handling():
+                    result = validate_result(await anext(generator))
+
+                    if items_to_parse:
+                        async for batch in handle_items_to_parse(result, items_to_parse_name, items_to_parse):
+                            yield batch
+                    else:
+                        yield result
+
+
+            except OceanAbortException as error:
+                errors.append(error)
+                ocean.metrics.inc_metric(
+                    name=MetricType.OBJECT_COUNT_NAME,
+                    labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT , MetricPhase.ExtractResult.FAILED],
+                    value=1
+                )
+    except StopAsyncIteration:
+        if errors:
+            raise ExceptionGroup(
+                "At least one of the resync generator iterations failed", errors
+            )
+
+
+def is_resource_supported(
+    kind: str, resync_event_mapping: dict[str | None, list[RESYNC_EVENT_LISTENER]]
+) -> bool:
+    return bool(resync_event_mapping[kind] or resync_event_mapping[None])
+
+def unsupported_kind_response(
+    kind: str, available_resync_kinds: list[str]
+) -> tuple[RESYNC_RESULT, list[Exception]]:
+    logger.error(f"Kind {kind} is not supported in this integration")
+    return [], [KindNotImplementedException(kind, available_resync_kinds)]
+
+class ProcessWrapper(multiprocessing.Process):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    async def join_async(self) -> None:
+        while self.exitcode is None:
+            await asyncio.sleep(2)
+        if self.exitcode != 0:
+            logger.error(f"Process {self.pid} failed with exit code {self.exitcode}")
+        else:
+            logger.info(f"Process {self.pid} finished with exit code {self.exitcode}")
+        return super().join()
+
+def clear_http_client_context() -> None:
+    try:
+        while _http_client.top is not None:
+            _http_client.pop()
+    except (RuntimeError, AttributeError):
+        pass
+
+    try:
+        while _port_http_client.top is not None:
+            _port_http_client.pop()
+    except (RuntimeError, AttributeError):
+        pass