port-ocean 0.27.5__tar.gz → 0.27.6__tar.gz

{port_ocean-0.27.5 → port_ocean-0.27.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.27.5
+Version: 0.27.6
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -22,12 +22,15 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
 Provides-Extra: cli
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
 Requires-Dist: aiostream (>=0.5.2,<0.7.0)
 Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
 Requires-Dist: confluent-kafka (>=2.10.1,<3.0.0)
 Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
+Requires-Dist: cryptography (>=44.0.1,<45.0.0)
 Requires-Dist: fastapi (>=0.116.0,<0.117.0)
 Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: ijson (>=3.4.0,<4.0.0)
 Requires-Dist: jinja2 (>=3.1.6)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
 Requires-Dist: jq (>=1.8.0,<2.0.0)

{port_ocean-0.27.5 → port_ocean-0.27.6}/integrations/_infra/Dockerfile.Deb

@@ -28,11 +28,14 @@ ARG INTEGRATION_VERSION
 ARG BUILD_CONTEXT
 ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
 ARG OAUTH_CONFIG_DIR=/app/.config
+ARG STREAMING_LOCATION=/tmp/ocean/streaming
 
 ENV LIBRDKAFKA_VERSION=2.8.2 \
-    PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
+    PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR} \
+    STREAMING_LOCATION=${STREAMING_LOCATION}
 
 RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
+RUN mkdir -p ${STREAMING_LOCATION}
 RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
 
 RUN mkdir -p ${OAUTH_CONFIG_DIR}

{port_ocean-0.27.5 → port_ocean-0.27.6}/integrations/_infra/Dockerfile.local

@@ -33,13 +33,15 @@ RUN apt-get update \
 
 ARG BUILD_CONTEXT
 ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
+ARG STREAMING_LOCATION=/tmp/ocean/streaming
 
 ENV PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
-
+ENV STREAMING_LOCATION=${STREAMING_LOCATION}
 # Create /tmp/ocean directory and set permissions
 
 
 RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
+RUN mkdir -p ${STREAMING_LOCATION}
 
 WORKDIR /app
 

{port_ocean-0.27.5 → port_ocean-0.27.6}/port_ocean/config/settings.py

@@ -73,6 +73,13 @@ class MetricsSettings(BaseOceanModel, extra=Extra.allow):
     webhook_url: str | None = Field(default=None)
 
 
+class StreamingSettings(BaseOceanModel, extra=Extra.allow):
+    enabled: bool = Field(default=False)
+    max_buffer_size_mb: int = Field(default=1024 * 1024 * 20)  # 20 mb
+    chunk_size: int = Field(default=1024 * 64)  # 64 kb
+    location: str = Field(default="/tmp/ocean/streaming")
+
+
 class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     _integration_config_model: BaseModel | None = None
 
@@ -114,6 +121,8 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     yield_items_to_parse: bool = False
     yield_items_to_parse_batch_size: int = 10
 
+    streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
+
     @validator("process_execution_mode")
     def validate_process_execution_mode(
         cls, process_execution_mode: ProcessExecutionMode

{port_ocean-0.27.5 → port_ocean-0.27.6}/port_ocean/helpers/async_client.py

@@ -4,6 +4,7 @@ import httpx
 from loguru import logger
 
 from port_ocean.helpers.retry import RetryTransport
+from port_ocean.helpers.stream import Stream
 
 
 class OceanAsyncClient(httpx.AsyncClient):
@@ -50,3 +51,9 @@ class OceanAsyncClient(httpx.AsyncClient):
             logger=logger,
             **(self._transport_kwargs or {}),
         )
+
+    async def get_stream(self, url: str, **kwargs: Any) -> Stream:
+        req = self.build_request("GET", url, **kwargs)
+        response = await self.send(req, stream=True)
+        response.raise_for_status()
+        return Stream(response)

port_ocean-0.27.6/port_ocean/helpers/stream.py

@@ -0,0 +1,71 @@
+import os
+from typing import Any, AsyncGenerator
+import uuid
+
+import aiofiles
+import httpx
+import ijson  # type: ignore[import-untyped]
+from cryptography.fernet import Fernet
+
+import port_ocean.context.ocean as ocean_context
+
+
+class Stream:
+    def __init__(self, response: httpx.Response):
+        self.response = response
+        self.headers = response.headers
+        self.status_code = response.status_code
+
+    async def _byte_stream(
+        self, chunk_size: int | None = None
+    ) -> AsyncGenerator[bytes, None]:
+        if chunk_size is None:
+            chunk_size = ocean_context.ocean.config.streaming.chunk_size
+
+        file_name = f"{ocean_context.ocean.config.streaming.location}/{uuid.uuid4()}"
+
+        crypt = Fernet(Fernet.generate_key())
+
+        try:
+            async for chunk in self.response.aiter_bytes(chunk_size=chunk_size):
+                async with aiofiles.open(f"{file_name}", "ab") as f:
+                    if len(chunk) > 0:
+                        await f.write(crypt.encrypt(chunk))
+                        await f.write(b"\n")
+        finally:
+            await self.response.aclose()
+
+        try:
+            async with aiofiles.open(f"{file_name}", mode="rb") as f:
+                while True:
+                    line = await f.readline()
+                    if not line:
+                        break
+                    data = crypt.decrypt(line)
+                    yield data
+        finally:
+            try:
+                os.remove(file_name)
+            except FileNotFoundError:
+                pass
+
+    async def get_json_stream(
+        self,
+        target_items: str = "",
+        max_buffer_size_mb: int | None = None,
+    ) -> AsyncGenerator[list[dict[str, Any]], None]:
+        if max_buffer_size_mb is None:
+            max_buffer_size_mb = ocean_context.ocean.config.streaming.max_buffer_size_mb
+
+        events = ijson.sendable_list()
+        coro = ijson.items_coro(events, target_items)
+        current_buffer_size = 0
+        async for chunk in self._byte_stream():
+            coro.send(chunk)
+            current_buffer_size += len(chunk)
+            if current_buffer_size >= max_buffer_size_mb:
+                if len(events) > 0:
+                    yield events
+                    events.clear()
+                    current_buffer_size = 0
+        yield events

{port_ocean-0.27.5 → port_ocean-0.27.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.27.5"
+version = "0.27.6"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"
@@ -65,6 +65,9 @@ cookiecutter = { version = "^2.1.1", optional = true }
 jinja2-time = { version = "^0.2.0", optional = true }
 prometheus-client = "^0.21.1"
 pytest-cov = "^6.0.0"
+ijson = "^3.4.0"
+aiofiles = "^24.1.0"
+cryptography = "^44.0.1"
 
 [tool.poetry.extras]
 cli = ["click", "rich", "cookiecutter", "jinja2-time"]
@@ -90,6 +93,7 @@ cryptography = ">=43.0.1,<45.0.0"
 psutil = "^7.0.0"
 memray = "^1.17.2"
 faker = "^37.3.0"
+types-aiofiles = "^24.1.0.20250809"
 
 [tool.towncrier]
 directory = "changelog"