port-ocean 0.27.3__py3-none-any.whl → 0.27.6__py3-none-any.whl

integrations/_infra/Dockerfile.Deb

@@ -28,11 +28,14 @@ ARG INTEGRATION_VERSION
 ARG BUILD_CONTEXT
 ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
 ARG OAUTH_CONFIG_DIR=/app/.config
+ARG STREAMING_LOCATION=/tmp/ocean/streaming
 
 ENV LIBRDKAFKA_VERSION=2.8.2 \
-    PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
+    PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR} \
+    STREAMING_LOCATION=${STREAMING_LOCATION}
 
 RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
+RUN mkdir -p ${STREAMING_LOCATION}
 RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
 
 RUN mkdir -p ${OAUTH_CONFIG_DIR}

integrations/_infra/Dockerfile.local

@@ -33,13 +33,15 @@ RUN apt-get update \
 
 ARG BUILD_CONTEXT
 ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
+ARG STREAMING_LOCATION=/tmp/ocean/streaming
 
 ENV PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
-
+ENV STREAMING_LOCATION=${STREAMING_LOCATION}
 # Create /tmp/ocean directory and set permissions
 
 
 RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
+RUN mkdir -p ${STREAMING_LOCATION}
 
 WORKDIR /app
 

port_ocean/config/settings.py

@@ -73,6 +73,13 @@ class MetricsSettings(BaseOceanModel, extra=Extra.allow):
     webhook_url: str | None = Field(default=None)
 
 
+class StreamingSettings(BaseOceanModel, extra=Extra.allow):
+    enabled: bool = Field(default=False)
+    max_buffer_size_mb: int = Field(default=1024 * 1024 * 20)  # 20 mb
+    chunk_size: int = Field(default=1024 * 64)  # 64 kb
+    location: str = Field(default="/tmp/ocean/streaming")
+
+
 class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     _integration_config_model: BaseModel | None = None
 
@@ -114,6 +121,8 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     yield_items_to_parse: bool = False
     yield_items_to_parse_batch_size: int = 10
 
+    streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
+
     @validator("process_execution_mode")
     def validate_process_execution_mode(
         cls, process_execution_mode: ProcessExecutionMode

port_ocean/core/event_listener/kafka.py

@@ -16,6 +16,7 @@ from port_ocean.core.event_listener.base import (
     EventListenerEvents,
     EventListenerSettings,
 )
+from pydantic import validator
 
 
 class KafkaEventListenerSettings(EventListenerSettings):
@@ -46,6 +47,19 @@ class KafkaEventListenerSettings(EventListenerSettings):
     kafka_security_enabled: bool = True
     consumer_poll_timeout: int = 1
 
+    @validator("brokers")
+    @classmethod
+    def parse_brokers(cls, v: str) -> str:
+        # If it's a JSON array string, parse and join
+        if v.strip().startswith("[") and v.strip().endswith("]"):
+            try:
+                parsed = json.loads(v)
+                if isinstance(parsed, list):
+                    return ",".join(parsed)
+            except json.JSONDecodeError:
+                pass
+        return v
+
     def get_changelog_destination_details(self) -> dict[str, Any]:
         """
         Returns the changelog destination configuration for the Kafka event listener.

port_ocean/helpers/async_client.py

@@ -4,6 +4,7 @@ import httpx
 from loguru import logger
 
 from port_ocean.helpers.retry import RetryTransport
+from port_ocean.helpers.stream import Stream
 
 
 class OceanAsyncClient(httpx.AsyncClient):
@@ -50,3 +51,9 @@ class OceanAsyncClient(httpx.AsyncClient):
             logger=logger,
             **(self._transport_kwargs or {}),
         )
+
+    async def get_stream(self, url: str, **kwargs: Any) -> Stream:
+        req = self.build_request("GET", url, **kwargs)
+        response = await self.send(req, stream=True)
+        response.raise_for_status()
+        return Stream(response)

port_ocean/helpers/stream.py

@@ -0,0 +1,71 @@
+import os
+from typing import Any, AsyncGenerator
+import uuid
+
+import aiofiles
+import httpx
+import ijson  # type: ignore[import-untyped]
+from cryptography.fernet import Fernet
+
+import port_ocean.context.ocean as ocean_context
+
+
+class Stream:
+    def __init__(self, response: httpx.Response):
+        self.response = response
+        self.headers = response.headers
+        self.status_code = response.status_code
+
+    async def _byte_stream(
+        self, chunk_size: int | None = None
+    ) -> AsyncGenerator[bytes, None]:
+        if chunk_size is None:
+            chunk_size = ocean_context.ocean.config.streaming.chunk_size
+
+        file_name = f"{ocean_context.ocean.config.streaming.location}/{uuid.uuid4()}"
+
+        crypt = Fernet(Fernet.generate_key())
+
+        try:
+            async for chunk in self.response.aiter_bytes(chunk_size=chunk_size):
+                async with aiofiles.open(f"{file_name}", "ab") as f:
+                    if len(chunk) > 0:
+                        await f.write(crypt.encrypt(chunk))
+                        await f.write(b"\n")
+        finally:
+            await self.response.aclose()
+
+        try:
+            async with aiofiles.open(f"{file_name}", mode="rb") as f:
+                while True:
+                    line = await f.readline()
+                    if not line:
+                        break
+                    data = crypt.decrypt(line)
+                    yield data
+        finally:
+            try:
+                os.remove(file_name)
+            except FileNotFoundError:
+                pass
+
+    async def get_json_stream(
+        self,
+        target_items: str = "",
+        max_buffer_size_mb: int | None = None,
+    ) -> AsyncGenerator[list[dict[str, Any]], None]:
+        if max_buffer_size_mb is None:
+            max_buffer_size_mb = ocean_context.ocean.config.streaming.max_buffer_size_mb
+
+        events = ijson.sendable_list()
+        coro = ijson.items_coro(events, target_items)
+        current_buffer_size = 0
+        async for chunk in self._byte_stream():
+            coro.send(chunk)
+            current_buffer_size += len(chunk)
+            if current_buffer_size >= max_buffer_size_mb:
+                if len(events) > 0:
+                    yield events
+                    events.clear()
+                    current_buffer_size = 0
+        yield events

port_ocean/tests/core/event_listener/test_kafka.py

@@ -0,0 +1,70 @@
+from port_ocean.core.event_listener.kafka import KafkaEventListenerSettings
+import pytest
+from pydantic import ValidationError
+
+
+def test_default_kafka_settings() -> None:
+    """Test default values are properly set"""
+    config = KafkaEventListenerSettings(type="KAFKA")
+    assert config.type == "KAFKA"
+    assert config.security_protocol == "SASL_SSL"
+    assert config.authentication_mechanism == "SCRAM-SHA-512"
+    assert config.kafka_security_enabled is True
+    assert config.consumer_poll_timeout == 1
+    assert "b-1-public.publicclusterprod" in config.brokers
+
+
+def test_brokers_json_array_parsing() -> None:
+    """Test that JSON array strings get converted to comma-separated"""
+    json_brokers = '["broker1:9092", "broker2:9092", "broker3:9092"]'
+    config = KafkaEventListenerSettings(type="KAFKA", brokers=json_brokers)
+    assert config.brokers == "broker1:9092,broker2:9092,broker3:9092"
+
+
+def test_brokers_regular_string_unchanged() -> None:
+    """Test that regular comma-separated strings pass through unchanged"""
+    regular_brokers = "broker1:9092,broker2:9092"
+    config = KafkaEventListenerSettings(type="KAFKA", brokers=regular_brokers)
+    assert config.brokers == regular_brokers
+
+
+def test_brokers_malformed_json_unchanged() -> None:
+    """Test that malformed JSON strings don't break validation"""
+    bad_json = "[broker1:9092, broker2:9092"
+    config = KafkaEventListenerSettings(type="KAFKA", brokers=bad_json)
+    assert config.brokers == bad_json
+
+
+def test_custom_values() -> None:
+    """Test overriding default values"""
+    config = KafkaEventListenerSettings(
+        type="KAFKA",
+        brokers="custom:9092",
+        security_protocol="PLAINTEXT",
+        authentication_mechanism="PLAIN",
+        kafka_security_enabled=False,
+        consumer_poll_timeout=5,
+    )
+    assert config.brokers == "custom:9092"
+    assert config.security_protocol == "PLAINTEXT"
+    assert config.authentication_mechanism == "PLAIN"
+    assert config.kafka_security_enabled is False
+    assert config.consumer_poll_timeout == 5
+
+
+def test_type_literal_validation() -> None:
+    """Test that type field only accepts KAFKA"""
+    with pytest.raises(ValidationError):
+        KafkaEventListenerSettings(type="RABBITMQ")  # type: ignore[arg-type]
+
+
+def test_empty_brokers_array() -> None:
+    """Test empty JSON array becomes empty string"""
+    config = KafkaEventListenerSettings(type="KAFKA", brokers="[]")
+    assert config.brokers == ""
+
+
+def test_single_broker_array() -> None:
+    """Test single broker in JSON array"""
+    config = KafkaEventListenerSettings(type="KAFKA", brokers='["single:9092"]')
+    assert config.brokers == "single:9092"

{port_ocean-0.27.3.dist-info → port_ocean-0.27.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.27.3
+Version: 0.27.6
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -22,12 +22,15 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
 Provides-Extra: cli
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
 Requires-Dist: aiostream (>=0.5.2,<0.7.0)
 Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
 Requires-Dist: confluent-kafka (>=2.10.1,<3.0.0)
 Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
-Requires-Dist: fastapi (>=0.115.3,<0.116.0)
+Requires-Dist: cryptography (>=44.0.1,<45.0.0)
+Requires-Dist: fastapi (>=0.116.0,<0.117.0)
 Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: ijson (>=3.4.0,<4.0.0)
 Requires-Dist: jinja2 (>=3.1.6)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
 Requires-Dist: jq (>=1.8.0,<2.0.0)

{port_ocean-0.27.3.dist-info → port_ocean-0.27.6.dist-info}/RECORD

@@ -1,9 +1,9 @@
-integrations/_infra/Dockerfile.Deb,sha256=ovmwNBRNrblTW6K9ru2FNTiT9qDgI7_zY28O__VMW6I,2367
+integrations/_infra/Dockerfile.Deb,sha256=weK3VrskDEsuvYaxerak-XIqDgTx_wPorAFQQjYH_FU,2493
 integrations/_infra/Dockerfile.alpine,sha256=7E4Sb-8supsCcseerHwTkuzjHZoYcaHIyxiBZ-wewo0,3482
 integrations/_infra/Dockerfile.base.builder,sha256=ESe1PKC6itp_AuXawbLI75k1Kruny6NTANaTinxOgVs,743
 integrations/_infra/Dockerfile.base.runner,sha256=uAcs2IsxrAAUHGXt_qULA5INr-HFguf5a5fCKiqEzbY,384
 integrations/_infra/Dockerfile.dockerignore,sha256=CM1Fxt3I2AvSvObuUZRmy5BNLSGC7ylnbpWzFgD4cso,1163
-integrations/_infra/Dockerfile.local,sha256=v-K0jgDj5vWyF4HCL8-wluUPB04pf9ycEjusRBmIVnc,1527
+integrations/_infra/Dockerfile.local,sha256=FFX9RvFqlaHvhUrRnnzUl0zQp2oKDFVRGkXJQPMQ7cI,1650
 integrations/_infra/Makefile,sha256=YgLKvuF_Dw4IA7X98Nus6zIW_3cJ60M1QFGs3imj5c4,2430
 integrations/_infra/README.md,sha256=ZtJFSMCTU5zTeM8ddRuW1ZL1ga8z7Ic2F3mxmgOSjgo,1195
 integrations/_infra/entry_local.sh,sha256=Sn2TexTEpruH2ixIAGsk-fZV6Y7pT3jd2Pi9TxBeFuw,633
@@ -70,7 +70,7 @@ port_ocean/clients/port/utils.py,sha256=osFyAjw7Y5Qf2uVSqC7_RTCQfijiL1zS74JJM0go
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=x1gFbzujrxn7EJudRT81C6eN9WsYAb3vOHwcpcpX8Tc,6370
 port_ocean/config/dynamic.py,sha256=Lrk4JRGtR-0YKQ9DDGexX5NGFE7EJ6VoHya19YYhssM,2687
-port_ocean/config/settings.py,sha256=-jgVUBiFbeVjd9GjkMH6cRLLQHhtvI9x1WNdVE5s_p0,7311
+port_ocean/config/settings.py,sha256=Zz_D40EXZEm0hzNdYgwdUy_s5LbJ6iMg3Zcl2n5NLUY,7686
 port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
 port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -87,7 +87,7 @@ port_ocean/core/event_listener/__init__.py,sha256=T3E52MKs79fNEW381p7zU9F2vOMvIi
 port_ocean/core/event_listener/base.py,sha256=GFBTHiYhCzps50phzopQFUlTGAluQkCRlyaRqOG4g1Y,2995
 port_ocean/core/event_listener/factory.py,sha256=M4Qi05pI840sjDIbdjUEgYe9Gp5ckoCkX-KgLBxUpZg,4096
 port_ocean/core/event_listener/http.py,sha256=_hkQmi9nNh8YG6hbfLrhkATsmGVO8y3qBWvrBHX5Nhk,2992
-port_ocean/core/event_listener/kafka.py,sha256=pVQfGtElIJN2P5lPBrMxyl05B8c2Q2wp5WvA6Pqjnyc,7501
+port_ocean/core/event_listener/kafka.py,sha256=7bUq4EoVv5sIBxiJnfhi7zdAI7whKoSIaS64Ic9B4Ys,7963
 port_ocean/core/event_listener/once.py,sha256=6DhxQLSz5hjEGhmgLU7aA2ZVRQw3tuwh8sh7N63gRdU,5855
 port_ocean/core/event_listener/polling.py,sha256=iZt59z4Dyus_wVvNfIPbq-G1CozAW9kfPU7uRDLPJVE,3604
 port_ocean/core/event_listener/webhooks_only.py,sha256=PTWnmbLtbJb3ySfotMpTWMYgDVy9zOSYIIGqNbWK0UU,1214
@@ -140,10 +140,11 @@ port_ocean/exceptions/port_defaults.py,sha256=2a7Koy541KxMan33mU-gbauUxsumG3NT4i
 port_ocean/exceptions/utils.py,sha256=gjOqpi-HpY1l4WlMFsGA9yzhxDhajhoGGdDDyGbLnqI,197
 port_ocean/exceptions/webhook_processor.py,sha256=4SnkVzVwiacH_Ip4qs1hRHa6GanhnojW_TLTdQQtm7Y,363
 port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-port_ocean/helpers/async_client.py,sha256=0MkFY46MuEYkju7gJ8HYeUTMBsjwAzESvATryJy2DaM,1698
+port_ocean/helpers/async_client.py,sha256=LOgUlZ5Cs_WUSc8XujCVjPGvzZ_3AuFJNKPy0FKV3fA,1987
 port_ocean/helpers/metric/metric.py,sha256=Aacz7bOd8ZCwEPpXAdwLbKRXf28Z4wiViG_GXiV_xWg,14529
 port_ocean/helpers/metric/utils.py,sha256=1lAgrxnZLuR_wUNDyPOPzLrm32b8cDdioob2lvnPQ1A,1619
 port_ocean/helpers/retry.py,sha256=VHAp6j9-Vid6aNR5sca3S0aW6b1S2oYw9vT9hi1N22U,18556
+port_ocean/helpers/stream.py,sha256=_UwsThzXynxWzL8OlBT1pmb2evZBi9HaaqeAGNuTuOI,2338
 port_ocean/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/log/handlers.py,sha256=ncVjgqrZRh6BhyRrA6DQG86Wsbxph1yWYuEC0cWfe-Q,3631
 port_ocean/log/logger_setup.py,sha256=0K3zVG0YYrYOWEV8-rCGks1o-bMRxgHXlqawu9w_tSw,2656
@@ -167,6 +168,7 @@ port_ocean/tests/config/test_config.py,sha256=Rk4N-ldVSOfn1p23NzdVdfqUpPrqG2cMut
 port_ocean/tests/conftest.py,sha256=JXASSS0IY0nnR6bxBflhzxS25kf4iNaABmThyZ0mZt8,101
 port_ocean/tests/core/conftest.py,sha256=5shShx81LRuQTBwS2sDIjNJO2LSD6cUNz46SgNYzjGY,7686
 port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcGsYZ2SCe_PYX5vLYM,5560
+port_ocean/tests/core/event_listener/test_kafka.py,sha256=PH90qk2fvdrQOSZD2QrvkGy8w_WoYb_KHGnqJ6PLHAo,2681
 port_ocean/tests/core/handlers/entities_state_applier/test_applier.py,sha256=7XWgwUB9uVYRov4VbIz1A-7n2YLbHTTYT-4rKJxjB0A,10711
 port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=TjSj8ssIqH23VJlO5PGovbudCqDbuE2-54iNQsD9K-I,14099
 port_ocean/tests/core/handlers/mixins/test_live_events.py,sha256=6yUsYooBYchiZP_eYa8PN1IhiztJShBdPguoseyNMzY,12482
@@ -205,8 +207,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=mMVq-1Ab5YpNiqN4PkiyTGlV_G0wkUDMMjTZp5z3pb0,1514
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.27.3.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.27.3.dist-info/METADATA,sha256=8q1fasdb2UPwIRETpBdDwk5m9IEDUOrBEBl5fSruMZ8,6887
-port_ocean-0.27.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.27.3.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.27.3.dist-info/RECORD,,
+port_ocean-0.27.6.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.27.6.dist-info/METADATA,sha256=KzFgTmio2jk9gaqUyvyZDhMIsyanGS28gXG0J-1gzdg,7015
+port_ocean-0.27.6.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.27.6.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.27.6.dist-info/RECORD,,