pam-python 0.2.16__tar.gz → 0.2.18__tar.gz

{pam_python-0.2.16/pam_python.egg-info → pam_python-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.2.16
+Version: 0.2.18
 Summary: Pam Python Library
 Author-email: Narongrit Kanhanoi <narongrit@pams.ai>
 Project-URL: Homepage, https://github.com/heart/pam-python
@@ -15,6 +15,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: Flask>=3.0.2
 Requires-Dist: requests>=2.32.3
+Requires-Dist: requests-toolbelt>=1.0.0
 Requires-Dist: PyYAML>=6.0.2
 Dynamic: license-file
 

{pam_python-0.2.16 → pam_python-0.2.18}/pam/__init__.py

@@ -1,3 +1,3 @@
 """PAM Data Plugin framework."""
 
-__version__ = "0.2.16"
+__version__ = "0.2.18"

{pam_python-0.2.16 → pam_python-0.2.18}/pam/api.py

@@ -2,6 +2,7 @@ from pathlib import Path
 import os
 import sqlite3
 import requests
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 from pam.models.sqlite_download import SQLiteDownloadResult, SQLiteDownloadStatus
 from pam.utils import log
 
@@ -36,7 +37,7 @@ class API:
 
     def http_upload(self, url: str, file_path: str, payload: dict = None) -> requests.Response | None:
         """
-        Uploads a file to the specified URL with optional form data (payload).
+        Streams a file as multipart form data without buffering the whole file.
 
         :param url: The URL to upload the file to.
         :param file_path: The path to the file to be uploaded.
@@ -49,11 +50,15 @@ class API:
 
         try:
             with open(file_path, 'rb') as file:
-                files = {'file': file}
+                fields = dict(payload or {})
+                fields['file'] = (Path(file_path).name, file)
+                # requests' files= buffers the entire multipart body. This encoder
+                # reads the open file incrementally and provides Content-Length.
+                body = MultipartEncoder(fields=fields)
                 response = self.session.post(
                     url,
-                    files=files,
-                    data=payload or {},  # optional form data
+                    data=body,
+                    headers={'Content-Type': body.content_type},
                     timeout=300
                 )
                 response.raise_for_status()

{pam_python-0.2.16 → pam_python-0.2.18}/pam/models/data_input_v2.py

@@ -71,21 +71,34 @@ class DataInputV2Dataset:
         for part in self.parts:
             yield from part.iter_csv_rows()
 
-    def scan_polars(self) -> Any:
-        """Return one lazy frame across all ordered CSV or Parquet parts."""
+    def scan_polars(self, empty_schema: dict[str, Any] | None = None) -> Any:
+        """Return one lazy frame across all ordered CSV or Parquet parts.
+
+        PAM represents a valid dataset with no rows as ``row_count=0`` and no
+        physical parts. In that case the plugin may provide its domain schema
+        so downstream computation can handle the empty input normally.
+        """
         pl = _import_polars()
         if not self.parts:
-            return pl.LazyFrame(schema={"id": pl.String})
+            schema = {"id": pl.String} if empty_schema is None else empty_schema
+            return pl.LazyFrame(schema=schema)
         scans = [part.scan_polars() for part in self.parts]
         if len(scans) == 1:
             return scans[0]
         return pl.concat(scans, how="diagonal_relaxed")
 
-    def read_pandas(self) -> Any:
-        """Materialize and concatenate all verified parts using their metadata."""
+    def read_pandas(self, empty_dtypes: dict[str, Any] | None = None) -> Any:
+        """Materialize and concatenate all verified parts using their metadata.
+
+        ``empty_dtypes`` is used only for a declared empty dataset, allowing
+        the plugin to retain its domain columns without fabricating data rows.
+        """
         pd = _import_pandas()
         if not self.parts:
-            return pd.DataFrame({"id": pd.Series(dtype="string")})
+            dtypes = {"id": "string"} if empty_dtypes is None else empty_dtypes
+            return pd.DataFrame(
+                {name: pd.Series(dtype=dtype) for name, dtype in dtypes.items()}
+            )
         frames = [part.read_pandas() for part in self.parts]
         if len(frames) == 1:
             return frames[0]

{pam_python-0.2.16 → pam_python-0.2.18}/pam/templates/init/AGENT.md

@@ -234,6 +234,11 @@ result = calculate(population, profile_frame, purchase_frame).collect()
 
 `scan_polars()` returns one lazy frame across ordered CSV or Parquet parts and
 uses verified manifest format metadata. CSV-only code may use `iter_rows()`.
+`row_count=0` with `parts=[]` is a valid empty dataset. The plugin decides the
+business result for the page population; it must not treat the absence of event
+rows as a transport error. If computation requires domain columns on an empty
+frame, call `scan_polars(empty_schema=...)` or
+`read_pandas(empty_dtypes=...)` with the plugin-owned schema.
 The external customer key is `id`. The population file is authoritative: in
 `contacts` mode, contacts with no event rows remain visible there. Use the
 opaque `data.next` cursor for the next `_request_data_v2(...)` call and stop when

{pam_python-0.2.16 → pam_python-0.2.18/pam_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.2.16
+Version: 0.2.18
 Summary: Pam Python Library
 Author-email: Narongrit Kanhanoi <narongrit@pams.ai>
 Project-URL: Homepage, https://github.com/heart/pam-python
@@ -15,6 +15,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: Flask>=3.0.2
 Requires-Dist: requests>=2.32.3
+Requires-Dist: requests-toolbelt>=1.0.0
 Requires-Dist: PyYAML>=6.0.2
 Dynamic: license-file
 

{pam_python-0.2.16 → pam_python-0.2.18}/pam_python.egg-info/requires.txt

@@ -1,3 +1,4 @@
 Flask>=3.0.2
 requests>=2.32.3
+requests-toolbelt>=1.0.0
 PyYAML>=6.0.2

{pam_python-0.2.16 → pam_python-0.2.18}/pyproject.toml

@@ -22,6 +22,7 @@ classifiers = [
 dependencies = [
     "Flask>=3.0.2",
     "requests>=2.32.3",
+    "requests-toolbelt>=1.0.0",
     "PyYAML>=6.0.2",
 ]
 

pam_python-0.2.18/tests/test_api.py

@@ -0,0 +1,102 @@
+import unittest
+import tempfile
+from pathlib import Path
+from email.parser import BytesParser
+from email.policy import default
+from unittest import mock
+
+import requests
+
+from pam.api import API
+
+
+class APITest(unittest.TestCase):
+    def test_http_upload_streams_file_and_preserves_multipart_fields(self):
+        api = API()
+        content = b'SQLite format 3\x00' * 65536
+        payload = {'token': 'test-token', 'file_name': 'state', 'is_shared': 'false'}
+        response = requests.Response()
+        response.status_code = 200
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'state.sqlite'
+            path.write_bytes(content)
+            real_open = open
+            opened = []
+
+            def guarded_open(*args, **kwargs):
+                file = real_open(*args, **kwargs)
+                opened.append(file)
+                original_read = file.read
+
+                def bounded_read(size=-1):
+                    self.assertGreaterEqual(size, 0)
+                    self.assertLessEqual(size, 16384)
+                    return original_read(size)
+
+                file.read = bounded_read
+                return file
+
+            def post(url, **kwargs):
+                self.assertNotIn('files', kwargs)
+                prepared = requests.Request('POST', url, **{
+                    key: kwargs[key] for key in ('data', 'headers')
+                }).prepare()
+                self.assertIs(prepared.body, kwargs['data'])
+                chunks = []
+                while chunk := prepared.body.read(16384):
+                    chunks.append(chunk)
+                body = b''.join(chunks)
+                self.assertEqual(int(prepared.headers['Content-Length']), len(body))
+                message = BytesParser(policy=default).parsebytes(
+                    ('Content-Type: ' + prepared.headers['Content-Type'] + '\r\n\r\n').encode() + body
+                )
+                parts = {part.get_param('name', header='content-disposition'): part
+                         for part in message.iter_parts()}
+                for key, value in payload.items():
+                    self.assertEqual(parts[key].get_payload(decode=True).decode(), value)
+                self.assertEqual(parts['file'].get_filename(), path.name)
+                self.assertEqual(parts['file'].get_payload(decode=True), content)
+                self.assertEqual(kwargs['timeout'], 300)
+                return response
+
+            api.session.post = mock.Mock(side_effect=post)
+            with mock.patch('pam.api.open', side_effect=guarded_open):
+                self.assertIs(api.http_upload('http://pam/sqlite_upload', str(path), payload), response)
+            self.assertTrue(opened[0].closed)
+            self.assertNotIn('file', payload)
+
+    def test_http_post_logs_error_response_body(self):
+        api = API()
+        response = requests.Response()
+        response.status_code = 404
+        response.url = "http://pam/api/dataadapter/adapter-1/log"
+        response._content = (
+            b'{"code":"NOT_FOUND","error_code":"ADAPTER_LOG_NOT_FOUND",'
+            b'"message":"Adapter execution log was not found"}'
+        )
+        api.session.post = mock.Mock(return_value=response)
+
+        with mock.patch("pam.api.log") as log_mock:
+            result = api.http_post(response.url, {"token": "secret"})
+
+        self.assertIsNone(result)
+        logged_message = log_mock.call_args.args[0]
+        self.assertIn("404 Client Error", logged_message)
+        self.assertIn('"error_code":"ADAPTER_LOG_NOT_FOUND"', logged_message)
+        self.assertNotIn("secret", logged_message)
+
+    def test_http_post_without_response_does_not_add_response_detail(self):
+        api = API()
+        api.session.post = mock.Mock(side_effect=requests.ConnectionError("offline"))
+
+        with mock.patch("pam.api.log") as log_mock:
+            result = api.http_post("http://pam/log", {})
+
+        self.assertIsNone(result)
+        logged_message = log_mock.call_args.args[0]
+        self.assertIn("offline", logged_message)
+        self.assertNotIn("Response:", logged_message)
+
+
+if __name__ == "__main__":
+    unittest.main()

{pam_python-0.2.16 → pam_python-0.2.18}/tests/test_data_input_v2.py

@@ -78,6 +78,30 @@ class DataInputV2Test(unittest.TestCase):
         self.assertEqual(0, data_input.datasets[0].row_count)
         self.assertEqual((), data_input.datasets[0].parts)
 
+    def test_empty_dataset_readers_use_plugin_owned_schema(self):
+        import polars as pl
+
+        manifest = self._valid_manifest()
+        manifest["datasets"][0]["row_count"] = 0
+        manifest["datasets"][0]["parts"] = []
+        self.dataset_path.unlink()
+        manifest_path = self._write_manifest(manifest)
+        dataset = parse_data_input_v2(
+            [str(manifest_path), str(self.population_path)]
+        ).datasets[0]
+
+        polars_frame = dataset.scan_polars(
+            empty_schema={"id": pl.String, "netamount": pl.Float64}
+        ).collect()
+        self.assertEqual(0, polars_frame.height)
+        self.assertEqual(["id", "netamount"], polars_frame.columns)
+
+        pandas_frame = dataset.read_pandas(
+            empty_dtypes={"id": "string", "netamount": "float64"}
+        )
+        self.assertTrue(pandas_frame.empty)
+        self.assertEqual(["id", "netamount"], pandas_frame.columns.tolist())
+
     def test_streams_population_and_ordered_dataset_parts(self):
         second_part_path = self._write_file(
             "dataset_001_demographic_part_0002.csv",

pam_python-0.2.16/tests/test_api.py

@@ -1,44 +0,0 @@
-import unittest
-from unittest import mock
-
-import requests
-
-from pam.api import API
-
-
-class APITest(unittest.TestCase):
-    def test_http_post_logs_error_response_body(self):
-        api = API()
-        response = requests.Response()
-        response.status_code = 404
-        response.url = "http://pam/api/dataadapter/adapter-1/log"
-        response._content = (
-            b'{"code":"NOT_FOUND","error_code":"ADAPTER_LOG_NOT_FOUND",'
-            b'"message":"Adapter execution log was not found"}'
-        )
-        api.session.post = mock.Mock(return_value=response)
-
-        with mock.patch("pam.api.log") as log_mock:
-            result = api.http_post(response.url, {"token": "secret"})
-
-        self.assertIsNone(result)
-        logged_message = log_mock.call_args.args[0]
-        self.assertIn("404 Client Error", logged_message)
-        self.assertIn('"error_code":"ADAPTER_LOG_NOT_FOUND"', logged_message)
-        self.assertNotIn("secret", logged_message)
-
-    def test_http_post_without_response_does_not_add_response_detail(self):
-        api = API()
-        api.session.post = mock.Mock(side_effect=requests.ConnectionError("offline"))
-
-        with mock.patch("pam.api.log") as log_mock:
-            result = api.http_post("http://pam/log", {})
-
-        self.assertIsNone(result)
-        logged_message = log_mock.call_args.args[0]
-        self.assertIn("offline", logged_message)
-        self.assertNotIn("Response:", logged_message)
-
-
-if __name__ == "__main__":
-    unittest.main()