port-ocean 0.10.6__tar.gz → 0.10.10__tar.gz

{port_ocean-0.10.6 → port_ocean-0.10.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.10.6
+Version: 0.10.10
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/cli/commands/new.py

@@ -9,6 +9,17 @@ from port_ocean.cli.commands.main import cli_start, print_logo, console
 from port_ocean.cli.utils import cli_root_path
 
 
+def symlink_makefile(result: str, name: str) -> None:
+    infra_make_file = os.path.join(
+        os.path.dirname(result), "../integrations/_infra/Makefile"
+    )
+    target_link_make_file = os.path.join(
+        os.path.dirname(result), f"../integrations/{name}/Makefile"
+    )
+
+    os.link(infra_make_file, target_link_make_file)
+
+
 def add_vscode_configuration(result: str, name: str) -> None:
     vscode_entry_root_path = "${workspaceFolder}/integrations/" + name
     new_vscode_entry = {
@@ -66,6 +77,7 @@ def new(path: str, is_private_integration: bool) -> None:
 
     if not is_private_integration:
         add_vscode_configuration(result, name)
+        symlink_makefile(result, name)
 
     console.print(
         "\n🌊 Ahoy, Captain! Your project is ready to set sail into the vast ocean of possibilities!",

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/clients/port/mixins/blueprints.py

@@ -64,7 +64,6 @@ class BlueprintClientMixin:
             f"Deleting blueprint with id: {identifier} with all entities: {delete_entities}"
         )
         headers = await self.auth.headers(user_agent_type)
-        response = None
 
         if not delete_entities:
             response = await self.client.delete(
@@ -82,7 +81,9 @@ class BlueprintClientMixin:
             handle_status_code(response, should_raise)
             return response.json().get("migrationId", "")
 
-    async def create_action(self, action: dict[str, Any]) -> None:
+    async def create_action(
+        self, action: dict[str, Any], should_log: bool = True
+    ) -> None:
         logger.info(f"Creating action: {action}")
         response = await self.client.post(
             f"{self.auth.api_url}/actions",
@@ -90,12 +91,13 @@ class BlueprintClientMixin:
             headers=await self.auth.headers(),
         )
 
-        handle_status_code(response)
+        handle_status_code(response, should_log=should_log)
 
     async def create_scorecard(
         self,
         blueprint_identifier: str,
         scorecard: dict[str, Any],
+        should_log: bool = True,
     ) -> None:
         logger.info(f"Creating scorecard: {scorecard}")
         response = await self.client.post(
@@ -104,11 +106,10 @@ class BlueprintClientMixin:
             headers=await self.auth.headers(),
         )
 
-        handle_status_code(response)
+        handle_status_code(response, should_log=should_log)
 
     async def create_page(
-        self,
-        page: dict[str, Any],
+        self, page: dict[str, Any], should_log: bool = True
     ) -> dict[str, Any]:
         logger.info(f"Creating page: {page}")
         response = await self.client.post(
@@ -117,7 +118,7 @@ class BlueprintClientMixin:
             headers=await self.auth.headers(),
         )
 
-        handle_status_code(response)
+        handle_status_code(response, should_log=should_log)
         return page
 
     async def delete_page(

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/clients/port/mixins/entities.py

@@ -29,7 +29,7 @@ class EntityClientMixin:
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
-    ) -> Entity:
+    ) -> Entity | None:
         validation_only = request_options["validation_only"]
         async with self.semaphore:
             logger.debug(
@@ -58,9 +58,16 @@ class EntityClientMixin:
             )
         handle_status_code(response, should_raise)
         result = response.json()
+
         result_entity = (
             Entity.parse_obj(result["entity"]) if result.get("entity") else entity
         )
+
+        # Happens when upsert fails and search identifier is defined.
+        # We return None to ignore the entity later in the delete process
+        if result_entity.is_using_search_identifier:
+            return None
+
         # In order to save memory we'll keep only the identifier, blueprint and relations of the
         # upserted entity result for later calculations
         reduced_entity = Entity(
@@ -94,7 +101,7 @@ class EntityClientMixin:
                 )
                 for entity in entities
             ),
-            return_exceptions=should_raise,
+            return_exceptions=True,
         )
         entity_results = [
             entity for entity in modified_entities_results if isinstance(entity, Entity)

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/clients/port/utils.py

@@ -18,7 +18,7 @@ if TYPE_CHECKING:
 # The max_keepalive_connections can't be too high, as it will cause the application to run out of available connections.
 PORT_HTTP_MAX_CONNECTIONS_LIMIT = 200
 PORT_HTTP_MAX_KEEP_ALIVE_CONNECTIONS = 50
-PORT_HTTP_TIMEOUT = 10.0
+PORT_HTTP_TIMEOUT = 60.0
 
 PORT_HTTPX_TIMEOUT = httpx.Timeout(PORT_HTTP_TIMEOUT)
 PORT_HTTPX_LIMITS = httpx.Limits(

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/config/settings.py

@@ -67,7 +67,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     allow_environment_variables_jq_access: bool = True
     initialize_port_resources: bool = True
     scheduled_resync_interval: int | None = None
-    client_timeout: int = 30
+    client_timeout: int = 60
     send_raw_data_examples: bool = True
     port: PortSettings
     event_listener: EventListenerSettingsType = Field(

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/core/defaults/initialize.py

@@ -120,7 +120,7 @@ async def _create_resources(
         )
         return
 
-    created_blueprints, errors = await gather_and_split_errors_from_results(
+    created_blueprints, blueprint_errors = await gather_and_split_errors_from_results(
         (
             port_client.create_blueprint(
                 blueprint, user_agent_type=UserAgentType.exporter
@@ -131,15 +131,17 @@ async def _create_resources(
 
     created_blueprints_identifiers = [bp["identifier"] for bp in created_blueprints]
 
-    if errors:
-        for error in errors:
+    if blueprint_errors:
+        for error in blueprint_errors:
             if isinstance(error, httpx.HTTPStatusError):
                 logger.warning(
                     f"Failed to create resources: {error.response.text}. Rolling back changes..."
                 )
 
-        raise AbortDefaultCreationError(created_blueprints_identifiers, errors)
-    created_pages_identifiers = []
+        raise AbortDefaultCreationError(
+            created_blueprints_identifiers, blueprint_errors
+        )
+
     try:
         for patch_stage in blueprint_patches:
             await asyncio.gather(
@@ -153,44 +155,43 @@ async def _create_resources(
                 )
             )
 
-        await asyncio.gather(
-            *(port_client.create_action(action) for action in defaults.actions)
+    except httpx.HTTPStatusError as err:
+        logger.error(f"Failed to create resources: {err.response.text}. continuing...")
+        raise AbortDefaultCreationError(created_blueprints_identifiers, [err])
+    try:
+        created_actions, actions_errors = await gather_and_split_errors_from_results(
+            (
+                port_client.create_action(action, should_log=False)
+                for action in defaults.actions
+            )
         )
 
-        await asyncio.gather(
-            *(
-                port_client.create_scorecard(blueprint_scorecards["blueprint"], action)
-                for blueprint_scorecards in defaults.scorecards
-                for action in blueprint_scorecards["data"]
+        created_scorecards, scorecards_errors = (
+            await gather_and_split_errors_from_results(
+                (
+                    port_client.create_scorecard(
+                        blueprint_scorecards["blueprint"], action, should_log=False
+                    )
+                    for blueprint_scorecards in defaults.scorecards
+                    for action in blueprint_scorecards["data"]
+                )
             )
         )
 
         created_pages, pages_errors = await gather_and_split_errors_from_results(
-            (port_client.create_page(page) for page in defaults.pages)
+            (port_client.create_page(page, should_log=False) for page in defaults.pages)
         )
-        created_pages_identifiers = [
-            page.get("identifier", "") for page in created_pages
-        ]
 
-        if pages_errors:
-            for error in pages_errors:
+        errors = actions_errors + scorecards_errors + pages_errors
+        if errors:
+            for error in errors:
                 if isinstance(error, httpx.HTTPStatusError):
                     logger.warning(
-                        f"Failed to create resources: {error.response.text}. Rolling back changes..."
+                        f"Failed to create resource: {error.response.text}. continuing..."
                     )
 
-            raise AbortDefaultCreationError(
-                created_blueprints_identifiers,
-                pages_errors,
-                created_pages_identifiers,
-            )
-    except httpx.HTTPStatusError as err:
-        logger.error(
-            f"Failed to create resources: {err.response.text}. Rolling back changes..."
-        )
-        raise AbortDefaultCreationError(
-            created_blueprints_identifiers, [err], created_pages_identifiers
-        )
+    except Exception as err:
+        logger.error(f"Failed to create resources: {err}. continuing...")
 
 
 async def _initialize_defaults(
@@ -227,19 +228,6 @@ async def _initialize_defaults(
                 for identifier in e.blueprints_to_rollback
             )
         )
-        if e.pages_to_rollback:
-            logger.warning(
-                f"Failed to create resources. Rolling back pages : {e.pages_to_rollback}"
-            )
-            await asyncio.gather(
-                *(
-                    port_client.delete_page(
-                        identifier,
-                    )
-                    for identifier in e.pages_to_rollback
-                )
-            )
-
         raise ExceptionGroup(str(e), e.errors)
 
 

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -106,18 +106,27 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 should_raise=False,
             )
         else:
+            entities_with_search_identifier: list[Entity] = []
+            entities_without_search_identifier: list[Entity] = []
+            for entity in entities:
+                if entity.is_using_search_identifier:
+                    entities_with_search_identifier.append(entity)
+                else:
+                    entities_without_search_identifier.append(entity)
+
             ordered_created_entities = reversed(
-                order_by_entities_dependencies(entities)
+                entities_with_search_identifier
+                + order_by_entities_dependencies(entities_without_search_identifier)
             )
             for entity in ordered_created_entities:
-                modified_entities.append(
-                    await self.context.port_client.upsert_entity(
-                        entity,
-                        event.port_app_config.get_port_request_options(),
-                        user_agent_type,
-                        should_raise=False,
-                    )
+                upsertedEntity = await self.context.port_client.upsert_entity(
+                    entity,
+                    event.port_app_config.get_port_request_options(),
+                    user_agent_type,
+                    should_raise=False,
                 )
+                if upsertedEntity:
+                    modified_entities.append(upsertedEntity)
         return modified_entities
 
     async def delete(

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/core/handlers/port_app_config/models.py

@@ -13,18 +13,22 @@ class Rule(BaseModel):
     value: str
 
 
-class SearchRelation(BaseModel):
+class IngestSearchQuery(BaseModel):
     combinator: str
-    rules: list[Rule | SearchRelation]
+    rules: list[Rule | IngestSearchQuery]
 
 
 class EntityMapping(BaseModel):
-    identifier: str
+    identifier: str | IngestSearchQuery
     title: str | None
     blueprint: str
     team: str | None
     properties: dict[str, str] = Field(default_factory=dict)
-    relations: dict[str, str | SearchRelation] = Field(default_factory=dict)
+    relations: dict[str, str | IngestSearchQuery] = Field(default_factory=dict)
+
+    @property
+    def is_using_search_identifier(self) -> bool:
+        return isinstance(self.identifier, dict)
 
 
 class MappingsConfig(BaseModel):

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -154,6 +154,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         results: list[RAW_ITEM],
         user_agent_type: UserAgentType,
     ) -> tuple[list[Entity], list[Exception]]:
+        if resource.port.entity.mappings.is_using_search_identifier:
+            logger.info(
+                f"Skip unregistering resource of kind {resource.kind}, as mapping defined with search identifier"
+            )
+            return [], []
+
         objects_diff = await self._calculate_raw([(resource, results)])
         entities_selector_diff, errors = objects_diff[0]
 
@@ -272,7 +278,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 [
                     entity
                     for entity in entities_to_delete
-                    if (entity.identifier, entity.blueprint)
+                    if not entity.is_using_search_identifier
+                    and (entity.identifier, entity.blueprint)
                     not in registered_entities_attributes
                 ],
             )

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/core/models.py

@@ -19,6 +19,10 @@ class Entity(BaseModel):
     properties: dict[str, Any] = {}
     relations: dict[str, Any] = {}
 
+    @property
+    def is_using_search_identifier(self) -> bool:
+        return isinstance(self.identifier, dict)
+
 
 class BlueprintRelation(BaseModel):
     many: bool

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/exceptions/port_defaults.py

@@ -6,10 +6,8 @@ class AbortDefaultCreationError(BaseOceanException):
         self,
         blueprints_to_rollback: list[str],
         errors: list[Exception],
-        pages_to_rollback: list[str] | None = None,
     ):
         self.blueprints_to_rollback = blueprints_to_rollback
-        self.pages_to_rollback = pages_to_rollback
         self.errors = errors
         super().__init__("Aborting defaults creation")
 

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/helpers/retry.py

@@ -290,6 +290,11 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
                 if remaining_attempts < 1:
                     self._log_error(request, error)
                     raise
+            except httpx.ReadTimeout as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
             except httpx.TimeoutException as e:
                 error = e
                 if remaining_attempts < 1:

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/middlewares.py

@@ -52,7 +52,14 @@ async def request_handler(
     request_id = generate_uuid()
 
     with logger.contextualize(request_id=request_id):
-        logger.bind(url=str(request.url), method=request.method).info("Request started")
+        log_level = (
+            "DEBUG"
+            if request.url.path == "/docs" or request.url.path == "/openapi.json"
+            else "INFO"
+        )
+        logger.bind(url=str(request.url), method=request.method).log(
+            log_level, f"Request to {request.url.path} started"
+        )
         response = await _handle_silently(call_next, request)
 
         end_time = get_time(seconds_precision=False)
@@ -61,5 +68,6 @@ async def request_handler(
         response.headers["X-Process-Time"] = str(time_elapsed)
         logger.bind(
             time_elapsed=time_elapsed, response_status=response.status_code
-        ).info("Request ended")
+        ).log(log_level, f"Request to {request.url.path} ended")
+
         return response

port_ocean-0.10.10/port_ocean/tests/clients/port/mixins/test_entities.py

@@ -0,0 +1,53 @@
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+
+from port_ocean.clients.port.mixins.entities import EntityClientMixin
+from port_ocean.core.models import Entity
+from httpx import ReadTimeout
+
+
+errored_entity_identifier: str = "a"
+expected_result_entities = [
+    Entity(identifier="b", blueprint="b"),
+    Entity(identifier="c", blueprint="c"),
+]
+all_entities = [
+    Entity(identifier=errored_entity_identifier, blueprint="a")
+] + expected_result_entities
+
+
+async def mock_upsert_entity(entity: Entity, *args: Any, **kwargs: Any) -> Entity:
+    if entity.identifier == errored_entity_identifier:
+        raise ReadTimeout("")
+    else:
+        return entity
+
+
+@pytest.fixture
+async def entity_client(monkeypatch: Any) -> EntityClientMixin:
+    # Arrange
+    entity_client = EntityClientMixin(auth=MagicMock(), client=MagicMock())
+    monkeypatch.setattr(entity_client, "upsert_entity", mock_upsert_entity)
+
+    return entity_client
+
+
+async def test_batch_upsert_entities_read_timeout_should_raise_false(
+    entity_client: EntityClientMixin,
+) -> None:
+    result_entities = await entity_client.batch_upsert_entities(
+        entities=all_entities, request_options=MagicMock(), should_raise=False
+    )
+
+    assert result_entities == expected_result_entities
+
+
+async def test_batch_upsert_entities_read_timeout_should_raise_true(
+    entity_client: EntityClientMixin,
+) -> None:
+    with pytest.raises(ReadTimeout):
+        await entity_client.batch_upsert_entities(
+            entities=all_entities, request_options=MagicMock(), should_raise=True
+        )

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/tests/helpers/__init__.py

@@ -1,5 +1,6 @@
 import sys
 from inspect import getmembers
+from pathlib import Path
 from typing import Dict, List, Set, Tuple, Union
 
 from yaml import safe_load
@@ -8,13 +9,17 @@ from port_ocean.bootstrap import create_default_app
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import RESYNC_RESULT
 from port_ocean.ocean import Ocean
-from port_ocean.utils.misc import load_module
+from port_ocean.utils.misc import get_spec_file, load_module
 
 
 def get_integration_ocean_app(integration_path: str) -> Ocean:
+    spec_file = get_spec_file(Path(integration_path))
+
+    config_factory = None if not spec_file else spec_file.get("configurations", [])
+
     default_app = create_default_app(
         integration_path,
-        None,
+        config_factory,
         {
             "port": {
                 "client_id": "bla",

{port_ocean-0.10.6 → port_ocean-0.10.10}/port_ocean/utils/async_iterators.py

@@ -36,7 +36,7 @@ async def stream_async_iterators_tasks(
     :return: A stream of results
     """
     if not tasks:
-        raise StopAsyncIteration("No tasks provided")
+        return
 
     if len(tasks) == 1:
         async for batch_items in tasks[0]:

{port_ocean-0.10.6 → port_ocean-0.10.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.10.6"
+version = "0.10.10"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"

port_ocean-0.10.6/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Makefile

@@ -1,74 +0,0 @@
-ACTIVATE := . .venv/bin/activate
-
-define run_checks
-	exit_code=0; \
-	cd $1; \
-	poetry check || exit_code=$$?;\
-	mypy . || exit_code=$$?; \
-	ruff check . || exit_code=$$?; \
-	black --check . || exit_code=$$?; \
-	if [ $$exit_code -eq 1 ]; then \
-		echo "\033[0;31mOne or more checks failed with exit code $$exit_code\033[0m"; \
-	else \
-		echo "\033[0;32mAll checks executed successfully.\033[0m"; \
-	fi; \
-	exit $$exit_code
-endef
-
-define install_poetry
-	if ! command -v poetry &> /dev/null; then \
-    	pip install --upgrade pip; \
-		pip install poetry; \
-	else \
-    	echo "Poetry is already installed."; \
-	fi
-endef
-
-define deactivate_virtualenv
-    if [ -n "$$VIRTUAL_ENV" ]; then \
-        unset VIRTUAL_ENV; \
-        unset PYTHONHOME; \
-        unset -f pydoc >/dev/null 2>&1; \
-        OLD_PATH="$$PATH"; \
-        PATH=$$(echo -n "$$PATH" | awk -v RS=: -v ORS=: '/\/virtualenv\/bin$$/ {next} {print}'); \
-        export PATH; \
-        hash -r; \
-        echo "Deactivated the virtual environment."; \
-    fi
-endef
-
-.SILENT: install install/prod lint run test clean
-
-install:
-	$(call deactivate_virtualenv) && \
-	$(call install_poetry) && \
-	poetry install --with dev
-
-install/prod:
-	$(call install_poetry) && \
-	poetry install --without dev --no-root --no-interaction --no-ansi --no-cache
-
-lint:
-	$(ACTIVATE) && \
-	$(call run_checks,.)
-
-run:
-	$(ACTIVATE) && ocean sail
-
-test:
-	$(ACTIVATE) && poetry run pytest -n auto
-
-clean:
-	@find . -name '.venv' -type d -exec rm -rf {} \;
-	@find . -name '*.pyc' -exec rm -rf {} \;
-	@find . -name '__pycache__' -exec rm -rf {} \;
-	@find . -name 'Thumbs.db' -exec rm -rf {} \;
-	@find . -name '*~' -exec rm -rf {} \;
-	rm -rf .cache
-	rm -rf build
-	rm -rf dist
-	rm -rf *.egg-info
-	rm -rf htmlcov
-	rm -rf .tox/
-	rm -rf docs/_build
-	rm -rf dist/