port-ocean 0.25.5__py3-none-any.whl → 0.26.2__py3-none-any.whl

integrations/_infra/Dockerfile.Deb

@@ -40,6 +40,7 @@ RUN apt-get update \
         openssl \
         curl \
         acl \
+        sudo \
     && apt-get clean
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}
@@ -68,6 +69,11 @@ RUN chmod a+x /app/init.sh
 RUN ln -s /app/.venv/bin/ocean /usr/bin/ocean
 # Add ocean user to ssl certs group
 RUN setfacl -m u:ocean:rwX /etc/ssl/certs
+
+# Allow ocean user to run update-ca-certificates without password (secure, limited sudo)
+RUN echo "ocean ALL=(root) NOPASSWD: /usr/sbin/update-ca-certificates" >> /etc/sudoers.d/ocean-certs \
+    && chmod 440 /etc/sudoers.d/ocean-certs
+
 USER ocean
 # Run the application
 CMD ["bash", "/app/init.sh"]

integrations/_infra/init.sh

@@ -1,5 +1,5 @@
 if test -e /usr/local/share/ca-certificates/cert.crt; then
-  update-ca-certificates
+  sudo update-ca-certificates
 fi
 
 ocean sail

port_ocean/clients/port/mixins/entities.py

@@ -483,38 +483,56 @@ class EntityClientMixin:
         query: dict[Any, Any] | None = None,
         parameters_to_include: list[str] | None = None,
     ) -> list[Entity]:
-        default_query = {
-            "combinator": "and",
-            "rules": [
-                {
-                    "property": "$datasource",
-                    "operator": "contains",
-                    "value": f"port-ocean/{self.auth.integration_type}/",
+        if query is None:
+            datasource_prefix = f"port-ocean/{self.auth.integration_type}/"
+            datasource_suffix = (
+                f"/{self.auth.integration_identifier}/{user_agent_type.value}"
+            )
+            logger.info(
+                f"Searching entities with datasource prefix: {datasource_prefix} and suffix: {datasource_suffix}"
+            )
+
+            response = await self.client.post(
+                f"{self.auth.api_url}/blueprints/entities/datasource-entities",
+                json={
+                    "datasource_prefix": datasource_prefix,
+                    "datasource_suffix": datasource_suffix,
                 },
-                {
-                    "property": "$datasource",
-                    "operator": "contains",
-                    "value": f"/{self.auth.integration_identifier}/{user_agent_type.value}",
+                headers=await self.auth.headers(user_agent_type),
+                extensions={"retryable": True},
+            )
+        else:
+            default_query = {
+                "combinator": "and",
+                "rules": [
+                    {
+                        "property": "$datasource",
+                        "operator": "contains",
+                        "value": f"port-ocean/{self.auth.integration_type}/",
+                    },
+                    {
+                        "property": "$datasource",
+                        "operator": "contains",
+                        "value": f"/{self.auth.integration_identifier}/{user_agent_type.value}",
+                    },
+                ],
+            }
+
+            if query.get("rules"):
+                query["rules"].extend(default_query["rules"])
+
+            logger.info(f"Searching entities with custom query: {query}")
+            response = await self.client.post(
+                f"{self.auth.api_url}/entities/search",
+                json=query,
+                headers=await self.auth.headers(user_agent_type),
+                params={
+                    "exclude_calculated_properties": "true",
+                    "include": parameters_to_include or ["blueprint", "identifier"],
                 },
-            ],
-        }
+                extensions={"retryable": True},
+            )
 
-        if query is None:
-            query = default_query
-        elif query.get("rules"):
-            query["rules"].append(default_query)
-
-        logger.info(f"Searching entities with query {query}")
-        response = await self.client.post(
-            f"{self.auth.api_url}/entities/search",
-            json=query,
-            headers=await self.auth.headers(user_agent_type),
-            params={
-                "exclude_calculated_properties": "true",
-                "include": parameters_to_include or ["blueprint", "identifier"],
-            },
-            extensions={"retryable": True},
-        )
         handle_port_status_code(response)
         return [Entity.parse_obj(result) for result in response.json()["entities"]]
 

port_ocean/clients/port/mixins/integrations.py

@@ -1,9 +1,10 @@
 import asyncio
-from typing import Any, Dict, List, TYPE_CHECKING, Optional, TypedDict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict
 from urllib.parse import quote_plus
 
 import httpx
 from loguru import logger
+
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.utils import handle_port_status_code
 from port_ocean.exceptions.port_defaults import DefaultsProvisionFailed
@@ -125,7 +126,7 @@ class IntegrationClientMixin:
 
         while attempts < INTEGRATION_POLLING_RETRY_LIMIT:
             logger.info(
-                f"Fetching created integration and validating config, attempt {attempts+1}/{INTEGRATION_POLLING_RETRY_LIMIT}",
+                f"Fetching created integration and validating config, attempt {attempts + 1}/{INTEGRATION_POLLING_RETRY_LIMIT}",
                 attempt=attempts,
             )
             response = await self._get_current_integration()
@@ -288,3 +289,18 @@ class IntegrationClientMixin:
         response = await self._delete_current_integration()
         handle_port_status_code(response, should_raise, should_log)
         return response.json()
+
+    async def post_integration_raw_data(
+        self, raw_data: list[dict[Any, Any]], sync_id: str, kind: str
+    ) -> None:
+        logger.debug("starting POST raw data request", raw_data=raw_data)
+        headers = await self.auth.headers()
+        response = await self.client.post(
+            f"{self.auth.api_url}/lakehouse/integration/{self.integration_identifier}/sync/{sync_id}/kind/{kind}/items",
+            headers=headers,
+            json={
+                "items": raw_data,
+            },
+        )
+        handle_port_status_code(response, should_log=False)
+        logger.debug("Finished POST raw data request")

port_ocean/config/settings.py

@@ -12,8 +12,8 @@ from port_ocean.core.event_listener import EventListenerSettingsType
 from port_ocean.core.models import (
     CachingStorageMode,
     CreatePortResourcesOrigin,
-    Runtime,
     ProcessExecutionMode,
+    Runtime,
 )
 from port_ocean.utils.misc import get_integration_name, get_spec_file
 
@@ -108,6 +108,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
 
     upsert_entities_batch_max_length: int = 20
     upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
+    lakehouse_enabled: bool = False
 
     @validator("process_execution_mode")
     def validate_process_execution_mode(

port_ocean/core/integrations/mixins/sync_raw.py

@@ -363,9 +363,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         results, errors = await self._get_resource_raw_results(resource_config)
         async_generators: list[ASYNC_GENERATOR_RESYNC_TYPE] = []
         raw_results: RAW_RESULT = []
+        lakehouse_data_enabled = await self._lakehouse_data_enabled()
+
         for result in results:
             if isinstance(result, dict):
                 raw_results.append(result)
+                if lakehouse_data_enabled:
+                    await ocean.port_client.post_integration_raw_data(result, event.id, resource_config.kind)
             else:
                 async_generators.append(result)
 
@@ -397,6 +401,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         for generator in async_generators:
             try:
                 async for items in generator:
+                    if lakehouse_data_enabled:
+                        await ocean.port_client.post_integration_raw_data(items, event.id, resource_config.kind)
                     number_of_raw_results += len(items)
                     if send_raw_data_examples_amount > 0:
                         send_raw_data_examples_amount = max(
@@ -462,6 +468,19 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
         return passed_entities, errors
 
+    async def _lakehouse_data_enabled(
+        self
+    ) -> bool:
+        """Check if lakehouse data is enabled.
+
+        Returns:
+            bool: True if lakehouse data is enabled, False otherwise
+        """
+        flags = await ocean.port_client.get_organization_feature_flags()
+        if "LAKEHOUSE_ELIGIBLE" in flags and ocean.config.lakehouse_enabled:
+            return True
+        return False
+
     async def register_raw(
         self,
         kind: str,

port_ocean/tests/clients/port/mixins/test_entities.py

@@ -1,11 +1,11 @@
-from typing import Any, List, Generator
-from unittest.mock import MagicMock, patch, AsyncMock
+from typing import Any, Generator, List
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
+from httpx import ReadTimeout
 
 from port_ocean.clients.port.mixins.entities import EntityClientMixin
 from port_ocean.core.models import Entity
-from httpx import ReadTimeout
 
 # Mock the ocean context at module level
 pytestmark = pytest.mark.usefixtures("mock_ocean")
@@ -172,3 +172,44 @@ async def test_batch_upsert_entities_read_timeout_should_raise_true(
             await entity_client.upsert_entities_in_batches(
                 entities=all_entities, request_options=MagicMock(), should_raise=True
             )
+
+
+async def test_search_entities_uses_datasource_route_when_query_is_none(
+    entity_client: EntityClientMixin,
+) -> None:
+    """Test that search_entities uses datasource route when query is None"""
+    mock_response = MagicMock()
+    mock_response.json.return_value = {"entities": []}
+    mock_response.is_error = False
+    mock_response.status_code = 200
+    mock_response.headers = {}
+    entity_client.client.post = AsyncMock(return_value=mock_response)  # type: ignore
+    entity_client.auth.headers = AsyncMock(  # type: ignore
+        return_value={"Authorization": "Bearer test"}
+    )
+
+    entity_client.auth.integration_type = "test-integration"
+    entity_client.auth.integration_identifier = "test-identifier"
+    entity_client.auth.api_url = "https://api.getport.io/v1"
+
+    mock_user_agent_type = MagicMock()
+    mock_user_agent_type.value = "sync"
+
+    await entity_client.search_entities(
+        user_agent_type=mock_user_agent_type,
+        query=None,
+    )
+
+    entity_client.client.post.assert_called_once()
+    call_args = entity_client.client.post.call_args
+
+    assert (
+        call_args[0][0]
+        == "https://api.getport.io/v1/blueprints/entities/datasource-entities"
+    )
+
+    expected_json = {
+        "datasource_prefix": "port-ocean/test-integration/",
+        "datasource_suffix": "/test-identifier/sync",
+    }
+    assert call_args[1]["json"] == expected_json

port_ocean/tests/core/conftest.py

@@ -1,10 +1,11 @@
 from contextlib import asynccontextmanager
 from typing import Any, AsyncGenerator
-from unittest.mock import MagicMock, AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from httpx import Response
 
+from port_ocean.cache.memory import InMemoryCacheProvider
 from port_ocean.clients.port.client import PortClient
 from port_ocean.config.settings import IntegrationSettings, MetricsSettings
 from port_ocean.context.event import EventContext
@@ -26,7 +27,6 @@ from port_ocean.core.handlers.port_app_config.models import (
 from port_ocean.core.models import Entity, ProcessExecutionMode
 from port_ocean.helpers.metric.metric import Metrics
 from port_ocean.ocean import Ocean
-from port_ocean.cache.memory import InMemoryCacheProvider
 
 
 @pytest.fixture
@@ -107,6 +107,7 @@ def mock_port_client(mock_http_client: MagicMock) -> PortClient:
     )
 
     mock_port_client.search_entities = AsyncMock(return_value=[])  # type: ignore
+    mock_port_client.get_organization_feature_flags = AsyncMock(return_value=[])  # type: ignore
     mock_port_client.client = mock_http_client
     return mock_port_client
 

{port_ocean-0.25.5.dist-info → port_ocean-0.26.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.25.5
+Version: 0.26.2
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.25.5.dist-info → port_ocean-0.26.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-integrations/_infra/Dockerfile.Deb,sha256=mGbUtGmNbA2nUqDmhoy5ehU-UpBxc4hlB8yutXjOpgU,1998
+integrations/_infra/Dockerfile.Deb,sha256=RW-BC8NaVmw3ASF7a3XNsjFN3ZN5gTVxbFrOmhBXq30,2251
 integrations/_infra/Dockerfile.alpine,sha256=7E4Sb-8supsCcseerHwTkuzjHZoYcaHIyxiBZ-wewo0,3482
 integrations/_infra/Dockerfile.base.builder,sha256=ESe1PKC6itp_AuXawbLI75k1Kruny6NTANaTinxOgVs,743
 integrations/_infra/Dockerfile.base.runner,sha256=uAcs2IsxrAAUHGXt_qULA5INr-HFguf5a5fCKiqEzbY,384
@@ -8,7 +8,7 @@ integrations/_infra/Makefile,sha256=YgLKvuF_Dw4IA7X98Nus6zIW_3cJ60M1QFGs3imj5c4,
 integrations/_infra/README.md,sha256=ZtJFSMCTU5zTeM8ddRuW1ZL1ga8z7Ic2F3mxmgOSjgo,1195
 integrations/_infra/entry_local.sh,sha256=Sn2TexTEpruH2ixIAGsk-fZV6Y7pT3jd2Pi9TxBeFuw,633
 integrations/_infra/grpcio.sh,sha256=m924poYznoRZ6Tt7Ct8Cs5AV_cmmOx598yIZ3z4DvZE,616
-integrations/_infra/init.sh,sha256=nN8lTrOhB286UfFvD6sJ9YJ-9asT9zVSddQB-RAb7Z4,99
+integrations/_infra/init.sh,sha256=yyAall_O2PHRA1qS5_Pm-kGm7HKlASLFfzmyWGic6vg,104
 port_ocean/__init__.py,sha256=uMpjg5d_cXgnyCxA_LmICR8zqBmC6Fe9Ivu9hcvJ7EY,313
 port_ocean/bootstrap.py,sha256=CN1M5pVecZ7z_Vfu86Dk2HjFMiuiwt6E_SSOLFCYRMk,1321
 port_ocean/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -60,8 +60,8 @@ port_ocean/clients/port/authentication.py,sha256=r7r8Ag9WuwXy-CmgeOoj-PHbmJAQxhb
 port_ocean/clients/port/client.py,sha256=dv0mxIOde6J-wFi1FXXZkoNPVHrZzY7RSMhNkDD9xgA,3566
 port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/mixins/blueprints.py,sha256=aMCG4zePsMSMjMLiGrU37h5z5_ElfMzTcTvqvOI5wXY,4683
-port_ocean/clients/port/mixins/entities.py,sha256=_FKoSEUxc_fGcdoWm4ZAeUKUnnzkGPMDQEsxmpbj8Vo,23352
-port_ocean/clients/port/mixins/integrations.py,sha256=DdMLxSHL2zmlLecnkJk1lWJ2fxkY89pSSL-m7H0zBuI,11256
+port_ocean/clients/port/mixins/entities.py,sha256=X2NqH00eK6TMJ3a3QEQRVQlKHzyj5l1FiPkIhonnbPg,24234
+port_ocean/clients/port/mixins/integrations.py,sha256=9G1vo3n9pG1t6siUmPdYtxXbfhGXKhWAWwKHr8x7tU4,11891
 port_ocean/clients/port/mixins/migrations.py,sha256=vdL_A_NNUogvzujyaRLIoZEu5vmKDY2BxTjoGP94YzI,1467
 port_ocean/clients/port/mixins/organization.py,sha256=A2cP5V49KnjoAXxjmnm_XGth4ftPSU0qURNfnyUyS_Y,1041
 port_ocean/clients/port/retry_transport.py,sha256=PtIZOAZ6V-ncpVysRUsPOgt8Sf01QLnTKB5YeKBxkJk,1861
@@ -70,7 +70,7 @@ port_ocean/clients/port/utils.py,sha256=osFyAjw7Y5Qf2uVSqC7_RTCQfijiL1zS74JJM0go
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=x1gFbzujrxn7EJudRT81C6eN9WsYAb3vOHwcpcpX8Tc,6370
 port_ocean/config/dynamic.py,sha256=Lrk4JRGtR-0YKQ9DDGexX5NGFE7EJ6VoHya19YYhssM,2687
-port_ocean/config/settings.py,sha256=R4Ju15XKbwQEg2W7uUCxoj4_9gUS9uYUFQnX-FUNRDI,7156
+port_ocean/config/settings.py,sha256=SaCophXTKj_MgJRw5vrIXaNds1NEfewt0jzPQ0ytM2o,7192
 port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
 port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -122,7 +122,7 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=AzlOARxsKY1YYQ6xhjYiDD-vELkci0Vz7THw8hDVgAU,38522
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=6AHuroSoqIKvdx2BAKtILxfQFeYB_wikYaM4JaTeoGw,39315
 port_ocean/core/integrations/mixins/utils.py,sha256=N76dNu1Y6UEg0_pcSdF4RO6dQIZ8EBfX3xMelgWdMxM,3779
 port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
 port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
@@ -159,12 +159,12 @@ port_ocean/tests/cache/test_memory_cache.py,sha256=xlwIOBU0RVLYYJU83l_aoZDzZ6QID
 port_ocean/tests/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/tests/clients/oauth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/tests/clients/oauth/test_oauth_client.py,sha256=2XVMQUalDpiD539Z7_dk5BK_ngXQzsTmb2lNBsfEm9c,3266
-port_ocean/tests/clients/port/mixins/test_entities.py,sha256=DeWbAQcaxT3LQQf_j9HA5nG7YgsQDvXmgK2aghlG9ug,6619
+port_ocean/tests/clients/port/mixins/test_entities.py,sha256=_ZEQT7UMzg1gW8kH8oFjgRVcLwQb7dFac48Tw_vcCqk,8018
 port_ocean/tests/clients/port/mixins/test_integrations.py,sha256=vRt_EMsLozQC1LJNXxlvnHj3-FlOBGgAYxg5T0IAqtA,7621
 port_ocean/tests/clients/port/mixins/test_organization_mixin.py,sha256=zzKYz3h8dl4Z5A2QG_924m0y9U6XTth1XYOfwNrd_24,914
 port_ocean/tests/config/test_config.py,sha256=Rk4N-ldVSOfn1p23NzdVdfqUpPrqG2cMut4Sv-sAOrw,1023
 port_ocean/tests/conftest.py,sha256=JXASSS0IY0nnR6bxBflhzxS25kf4iNaABmThyZ0mZt8,101
-port_ocean/tests/core/conftest.py,sha256=7K_M1--wQ08VmiQRB0vo1nst2X00cwsuBS5UfERsnG8,7589
+port_ocean/tests/core/conftest.py,sha256=5shShx81LRuQTBwS2sDIjNJO2LSD6cUNz46SgNYzjGY,7686
 port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcGsYZ2SCe_PYX5vLYM,5560
 port_ocean/tests/core/handlers/entities_state_applier/test_applier.py,sha256=_CZyViY9_gnxjY6ogWcDdmEDuejvpALogf9ESjVAwFY,10675
 port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=AyzLclJFKz8YzSi2hiFYVdXRzCFmqQwTxnwNFTo9roU,14091
@@ -203,8 +203,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.25.5.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.25.5.dist-info/METADATA,sha256=z6eqWLmVOCtooAj5NDNhgK76ZNsAtBXWufhYAz2ee80,6887
-port_ocean-0.25.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.25.5.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.25.5.dist-info/RECORD,,
+port_ocean-0.26.2.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.26.2.dist-info/METADATA,sha256=vKDMPtrSkiVpDBs1OdmO3RdRYqp88fLfKuQdXPL_no0,6887
+port_ocean-0.26.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.26.2.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.26.2.dist-info/RECORD,,