odsbox 1.0.16rc492.post1__tar.gz → 1.0.16rc494.post1__tar.gz

{odsbox-1.0.16rc492.post1 → odsbox-1.0.16rc494.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: odsbox
-Version: 1.0.16rc492.post1
+Version: 1.0.16rc494.post1
 Summary: Toolbox for accessing ASAM ODS servers using the HTTP API
 Keywords: asam,ods,measurement,data access,data science,data analysis
 Author-email: Andreas Krantz <a.krantz@peak-solution.de>

{odsbox-1.0.16rc492.post1 → odsbox-1.0.16rc494.post1}/docs/requirements.txt

@@ -5,3 +5,4 @@ sphinx-copybutton
 sphinx-design
 sphinx-rtd-theme
 sphinx-sitemap
+ipython

{odsbox-1.0.16rc492.post1 → odsbox-1.0.16rc494.post1}/src/odsbox/__init__.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-__version__ = "1.0.16""-rc492-post1"
+__version__ = "1.0.16""-rc494-post1"
 
 if TYPE_CHECKING:
     from .con_i import ConI

{odsbox-1.0.16rc492.post1 → odsbox-1.0.16rc494.post1}/src/odsbox/con_i.py

@@ -272,10 +272,10 @@ class ConI:
                                     This uses pandas native nullable data types for better type preservation.
                                     Defaults to True.
         :param str result_naming_mode: Controls how result column names are generated.
-                                        "query" (default): Uses column names from the JAQUEL query
-                                                          (e.g., 'name', 'phys_dimension.name').
-                                        "model": Uses column names from the ods.Model schema
-                                                (e.g., 'Unit.Name', 'PhysDimension.Name').
+                                       "query" (default): Uses column names from the JAQUEL query
+                                       (e.g., 'name', 'phys_dimension.name').
+                                       "model": Uses column names from the ods.Model schema
+                                       (e.g., 'Unit.Name', 'PhysDimension.Name').
         :param kwargs: Additional arguments passed to `to_pandas`.
         :raises requests.HTTPError: If query fails.
         :return DataFrame: The DataMatrices as Pandas.DataFrame with columns named according to `result_naming_mode`.
@@ -691,6 +691,7 @@ class ConI:
         target_file_or_folder: str,
         overwrite_existing: bool = False,
         default_filename: str = "download.bin",
+        chunk_size: int = 8192,
     ) -> str:
         """
         Read file content from server.
@@ -701,6 +702,7 @@ class ConI:
         :param bool overwrite_existing: If existing files should be overwritten. It defaults to False.
         :param str default_filename: Default filename if no filename is provided by server.
                                      It defaults to "download.bin".
+        :param int chunk_size: Size of chunks in bytes to stream. It defaults to 8192 (8KB).
         :raises requests.HTTPError: If something went wrong.
         :raises FileExistsError: If file already exists and 'overwrite_existing' is False.
         :raises ValueError: If no open session.
@@ -721,6 +723,7 @@ class ConI:
             },
             timeout=self.__request_timeout,
             allow_redirects=self.__allow_redirects,
+            stream=True,
         )
         self.check_requests_response(file_response)
 
@@ -740,7 +743,9 @@ class ConI:
             raise FileExistsError(f"File '{target_file_path}' already exists and 'overwrite_existing' is False.")
 
         with open(target_file_path, "wb") as file:
-            file.write(file_response.content)
+            for chunk in file_response.iter_content(chunk_size=chunk_size):
+                if chunk:  # filter out keep-alive new chunks
+                    file.write(chunk)
 
         return target_file_path
 

odsbox-1.0.16rc494.post1/tests/test_con_i_chunked_download.py

@@ -0,0 +1,214 @@
+"""Tests for chunked download functionality in file_access_download"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from unittest import mock
+
+import pytest
+
+from odsbox.con_i import ConI
+from odsbox.proto.ods_pb2 import FileIdentifier
+
+
+@pytest.fixture
+def con_i():
+    return ConI("https://docker.peak-solution.de:10032/api", ("Demo", "mdm"))
+
+
+class TestChunkedDownload:
+    """Test suite for chunked download functionality"""
+
+    def test_chunked_download_with_default_chunk_size(self, con_i):
+        """Test that default chunk size (8192 bytes) is used when not specified"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            target_file = os.path.join(temp_dir, "test_file.bin")
+
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=[b"chunk1", b"chunk2"])
+                    mock_response.headers = {}
+                    mock_get.return_value = mock_response
+
+                    con_i.file_access_download(file_identifier, target_file, overwrite_existing=True)
+
+                    # Verify iter_content was called with default chunk size
+                    mock_response.iter_content.assert_called_once_with(chunk_size=8192)
+
+    def test_chunked_download_with_custom_chunk_size(self, con_i):
+        """Test that custom chunk size is used when specified"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        custom_chunk_size = 4096
+        with tempfile.TemporaryDirectory() as temp_dir:
+            target_file = os.path.join(temp_dir, "test_file.bin")
+
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=[b"chunk1", b"chunk2"])
+                    mock_response.headers = {}
+                    mock_get.return_value = mock_response
+
+                    con_i.file_access_download(
+                        file_identifier, target_file, overwrite_existing=True, chunk_size=custom_chunk_size
+                    )
+
+                    # Verify iter_content was called with custom chunk size
+                    mock_response.iter_content.assert_called_once_with(chunk_size=custom_chunk_size)
+
+    def test_chunked_download_writes_all_chunks(self, con_i):
+        """Test that all chunks are written to file"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        chunks = [b"chunk1_", b"chunk2_", b"chunk3_"]
+        expected_content = b"chunk1_chunk2_chunk3_"
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            target_file = os.path.join(temp_dir, "test_file.bin")
+
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=chunks)
+                    mock_response.headers = {}
+                    mock_get.return_value = mock_response
+
+                    con_i.file_access_download(file_identifier, target_file, overwrite_existing=True)
+
+                    # Verify file content matches all chunks
+                    with open(target_file, "rb") as f:
+                        assert f.read() == expected_content
+
+    def test_chunked_download_filters_empty_chunks(self, con_i):
+        """Test that empty chunks (keep-alive chunks) are filtered out"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        # Include empty byte strings (keep-alive chunks)
+        chunks = [b"data1", b"", b"data2", b"", b"data3"]
+        expected_content = b"data1data2data3"
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            target_file = os.path.join(temp_dir, "test_file.bin")
+
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=chunks)
+                    mock_response.headers = {}
+                    mock_get.return_value = mock_response
+
+                    con_i.file_access_download(file_identifier, target_file, overwrite_existing=True)
+
+                    # Verify that only non-empty chunks were written
+                    with open(target_file, "rb") as f:
+                        assert f.read() == expected_content
+
+    def test_chunked_download_with_stream_true(self, con_i):
+        """Test that stream=True is set in the HTTP request"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            target_file = os.path.join(temp_dir, "test_file.bin")
+
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=[b"data"])
+                    mock_response.headers = {}
+                    mock_get.return_value = mock_response
+
+                    con_i.file_access_download(file_identifier, target_file, overwrite_existing=True)
+
+                    # Verify stream=True is in the call
+                    call_kwargs = mock_get.call_args[1]
+                    assert call_kwargs["stream"] is True
+
+    def test_chunked_download_large_file(self, con_i):
+        """Test downloading a large file with chunked streaming"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        # Simulate a 1MB file downloaded in 8KB chunks
+        chunk_size = 8192
+        num_chunks = 128
+        chunks = [b"x" * chunk_size for _ in range(num_chunks)]
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            target_file = os.path.join(temp_dir, "large_file.bin")
+
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=chunks)
+                    mock_response.headers = {}
+                    mock_get.return_value = mock_response
+
+                    con_i.file_access_download(file_identifier, target_file, overwrite_existing=True)
+
+                    # Verify file size is correct
+                    file_size = os.path.getsize(target_file)
+                    assert file_size == chunk_size * num_chunks
+
+    def test_chunked_download_with_content_disposition(self, con_i):
+        """Test chunked download respects Content-Disposition header"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        expected_filename = "server_filename.bin"
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=[b"data"])
+                    mock_response.headers = {"Content-Disposition": f'attachment; filename="{expected_filename}"'}
+                    mock_get.return_value = mock_response
+
+                    result_path = con_i.file_access_download(file_identifier, temp_dir, overwrite_existing=True)
+
+                    # Verify filename matches Content-Disposition
+                    assert os.path.basename(result_path) == expected_filename
+                    with open(result_path, "rb") as f:
+                        assert f.read() == b"data"
+
+    def test_chunked_download_preserves_binary_data(self, con_i):
+        """Test that binary data integrity is preserved through chunked download"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        # Create binary data with all byte values
+        binary_data = bytes(range(256))
+        chunks = [binary_data[i : i + 16] for i in range(0, len(binary_data), 16)]
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            target_file = os.path.join(temp_dir, "binary_file.bin")
+
+            with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                    mock_response = mock.Mock()
+                    mock_response.iter_content = mock.Mock(return_value=chunks)
+                    mock_response.headers = {}
+                    mock_get.return_value = mock_response
+
+                    con_i.file_access_download(file_identifier, target_file, overwrite_existing=True)
+
+                    # Verify binary data is intact
+                    with open(target_file, "rb") as f:
+                        assert f.read() == binary_data
+
+    def test_chunked_download_chunk_size_variations(self, con_i):
+        """Test various chunk sizes"""
+        file_identifier = FileIdentifier(aid=4711, iid=1)
+        chunk_sizes = [1024, 4096, 8192, 16384, 65536]
+
+        for chunk_size in chunk_sizes:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                target_file = os.path.join(temp_dir, "test_file.bin")
+
+                with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
+                    with mock.patch.object(con_i._ConI__session, "get") as mock_get:
+                        mock_response = mock.Mock()
+                        mock_response.iter_content = mock.Mock(return_value=[b"x" * chunk_size])
+                        mock_response.headers = {}
+                        mock_get.return_value = mock_response
+
+                        con_i.file_access_download(
+                            file_identifier, target_file, overwrite_existing=True, chunk_size=chunk_size
+                        )
+
+                        # Verify correct chunk size was requested
+                        mock_response.iter_content.assert_called_once_with(chunk_size=chunk_size)

{odsbox-1.0.16rc492.post1 → odsbox-1.0.16rc494.post1}/tests/test_con_i_file_access.py

@@ -25,7 +25,7 @@ def test_file_access_download_success(con_i):
     with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
         with mock.patch.object(con_i._ConI__session, "get") as mock_get:
             mock_response = mock.Mock()
-            mock_response.content = b"file content"
+            mock_response.iter_content = mock.Mock(return_value=[b"file content"])
             mock_response.headers = {"Content-Disposition": "attachment; filename=test_download.bin"}
             mock_get.return_value = mock_response
 
@@ -46,7 +46,7 @@ def test_file_access_download_success_2(con_i):
     with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
         with mock.patch.object(con_i._ConI__session, "get") as mock_get:
             mock_response = mock.Mock()
-            mock_response.content = b"file content"
+            mock_response.iter_content = mock.Mock(return_value=[b"file content"])
             mock_response.headers = {"Content-Disposition": 'attachment; filename="test_download.bin"'}
             mock_get.return_value = mock_response
 
@@ -68,7 +68,7 @@ def test_file_access_download_success_3(con_i):
         with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
             with mock.patch.object(con_i._ConI__session, "get") as mock_get:
                 mock_response = mock.Mock()
-                mock_response.content = b"file content"
+                mock_response.iter_content = mock.Mock(return_value=[b"file content"])
                 mock_response.headers = {"Content-Disposition": 'attachment; filename="test_download.bin"'}
                 mock_get.return_value = mock_response
 
@@ -88,7 +88,7 @@ def test_file_access_download_success_4(con_i):
         with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
             with mock.patch.object(con_i._ConI__session, "get") as mock_get:
                 mock_response = mock.Mock()
-                mock_response.content = b"file content"
+                mock_response.iter_content = mock.Mock(return_value=[b"file content"])
                 mock_response.headers = {}
                 mock_get.return_value = mock_response
 
@@ -110,7 +110,7 @@ def test_file_access_download_no_overwrite(con_i):
     with mock.patch.object(con_i, "file_access", return_value="http://example.com/file"):
         with mock.patch.object(con_i._ConI__session, "get") as mock_get:
             mock_response = mock.Mock()
-            mock_response.content = b"new file content"
+            mock_response.iter_content = mock.Mock(return_value=[b"new file content"])
             mock_response.headers = {"Content-Disposition": "attachment; filename=test_download_no_overwrite.bin"}
             mock_get.return_value = mock_response
 

odsbox-1.0.16rc492.post1/SECURITY.md

@@ -1,41 +0,0 @@
-<!-- BEGIN MICROSOFT SECURITY.MD V0.0.5 BLOCK -->
-
-## Security
-
-Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
-
-If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below.
-
-## Reporting Security Issues
-
-**Please do not report security vulnerabilities through public GitHub issues.**
-
-Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
-
-If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
-
-You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
-
-Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
-
-  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
-  * Full paths of source file(s) related to the manifestation of the issue
-  * The location of the affected source code (tag/branch/commit or direct URL)
-  * Any special configuration required to reproduce the issue
-  * Step-by-step instructions to reproduce the issue
-  * Proof-of-concept or exploit code (if possible)
-  * Impact of the issue, including how an attacker might exploit the issue
-
-This information will help us triage your report more quickly.
-
-If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.
-
-## Preferred Languages
-
-We prefer all communications to be in English.
-
-## Policy
-
-Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).
-
-<!-- END MICROSOFT SECURITY.MD BLOCK -->