pam-python 0.2.19__tar.gz → 0.2.20__tar.gz

{pam_python-0.2.19/pam_python.egg-info → pam_python-0.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.2.19
+Version: 0.2.20
 Summary: Pam Python Library
 Author-email: Narongrit Kanhanoi <narongrit@pams.ai>
 Project-URL: Homepage, https://github.com/heart/pam-python

{pam_python-0.2.19 → pam_python-0.2.20}/pam/__init__.py

@@ -1,3 +1,3 @@
 """PAM Data Plugin framework."""
 
-__version__ = "0.2.19"
+__version__ = "0.2.20"

{pam_python-0.2.19 → pam_python-0.2.20}/pam/models/data_input_v2.py

@@ -127,6 +127,7 @@ class DataInputV2:
     datasets: tuple[DataInputV2Dataset, ...]
     run_complete: bool
     next: str
+    missing_contacts: DataInputV2File | None = None
 
     def iter_population_rows(self) -> Iterator[dict[str, str]]:
         """Stream the population file for this customer-aligned logical page."""
@@ -210,6 +211,15 @@ def parse_data_input_v2(input_files: list[str]) -> DataInputV2:
     if contact_count > max_contacts_per_batch:
         raise ValueError("V2 population contact_count exceeds configured batch maximum")
 
+    missing_contacts = None
+    if "missing_contacts" in manifest:
+        missing_contacts = _parse_and_verify_file(
+            manifest["missing_contacts"],
+            files_by_name,
+            declared_names,
+            population_format,
+        )
+
     raw_datasets = manifest.get("datasets")
     if not isinstance(raw_datasets, list) or len(raw_datasets) > V2_MAX_DATASETS:
         raise ValueError("V2 manifest datasets must be a list of at most five items")
@@ -283,6 +293,7 @@ def parse_data_input_v2(input_files: list[str]) -> DataInputV2:
         max_contacts_per_batch=max_contacts_per_batch,
         time_window=time_window,
         population=population_file,
+        missing_contacts=missing_contacts,
         datasets=tuple(datasets),
         run_complete=_required_bool(manifest, "run_complete"),
         next=_required_optional_string(manifest, "next"),

{pam_python-0.2.19 → pam_python-0.2.20}/pam/task_manager.py

@@ -9,6 +9,7 @@ import math
 import os
 import time
 import threading
+import uuid
 from typing import Dict, TYPE_CHECKING
 from pam.utils import log
 from pam.api import API
@@ -453,34 +454,31 @@ class TaskManager(ITaskManager):
         return response
 
     def service_upload_report(self, service: Service, file_path):
-        """
-        Uploads a report file asynchronously and logs the response.
-        """
-        endpoint = service.request.response_api
-
-        def handle_report_response(response):
-            """Logs the response after the report upload completes."""
-            if response is None:
-                log(f"Response from report upload to {endpoint}: None")
-                return
-            try:
-                response_data = response.json()
-            except ValueError:
-                response_data = response.text
-            log(f"Response from report upload to {endpoint}: {response_data}")
-
-        def upload_wrapper():
-            """Wrapper for the report upload to handle response logging."""
-            response = self.api.http_upload(
-                endpoint,
-                file_path,
-                {"token": service.request.token},
-            )
-            handle_report_response(response)
-
-        log(f"Uploading Report to: {endpoint}")
-        http_thread = threading.Thread(target=upload_wrapper)
-        http_thread.start()
+        """Wait for PAM to enqueue a legacy report; propagate delivery failures."""
+        trace_id = uuid.uuid4().hex
+        started = time.monotonic()
+        log(f"report_delivery_started trace_id={trace_id} service={service.request.service_name} "
+            f"report_file={Path(file_path).name}")
+        response = self.api.http_upload(
+            service.request.response_api,
+            file_path,
+            {"token": service.request.token, "report_delivery_id": trace_id},
+        )
+        status = response.status_code if response is not None else None
+        if response is None or not 200 <= status < 300:
+            log(f"report_delivery_failed trace_id={trace_id} http_status={status}")
+            raise RuntimeError(f"Report delivery failed trace_id={trace_id} http_status={status}")
+        try:
+            body = response.json()
+        except ValueError:
+            body = {}
+        stage = body.get("stage", "accepted_unconfirmed") if isinstance(body, dict) else "accepted_unconfirmed"
+        log(f"report_delivery_response trace_id={trace_id} http_status={status} "
+            f"stage={stage} elapsed_s={time.monotonic() - started:.3f}")
+        # Older PAM deployments only acknowledge receipt. Do not report that as
+        # successful publication: the new endpoint explicitly confirms enqueue.
+        if not isinstance(body, dict) or body.get("report_delivery_id") != trace_id or stage != "tracker_queued":
+            raise RuntimeError(f"Report delivery was not confirmed; update PAM first trace_id={trace_id}")
 
     def service_publish_report_pointer_v2(
         self,

{pam_python-0.2.19 → pam_python-0.2.20}/pam/templates/init/AGENT.md

@@ -680,3 +680,13 @@ Prefer streaming/chunked/incremental processing whenever possible.
 - Treat Polars lazy queries as the default compute path.
 - Use Pandas only when the selected service template or an integration requires it.
 - Prefer streaming or incremental durable steps when a query cannot stay in memory.
+
+### Missing contacts in V2 input
+
+`data.missing_contacts` is an optional verified `DataInputV2File` with one `id`
+column, using the same CSV or Parquet format as population. Read it with
+`scan_polars()` or `read_pandas()`; `iter_csv_rows()` is CSV-only. Older
+manifests expose `None`, while a present zero-row file is valid. These IDs are
+excluded from population and configured datasets. Their absence does not prove
+deletion; plugin developers decide whether and how to handle them. Read the
+file before page cleanup. Do not treat it as a configured dataset position.

{pam_python-0.2.19 → pam_python-0.2.20/pam_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.2.19
+Version: 0.2.20
 Summary: Pam Python Library
 Author-email: Narongrit Kanhanoi <narongrit@pams.ai>
 Project-URL: Homepage, https://github.com/heart/pam-python

{pam_python-0.2.19 → pam_python-0.2.20}/pam_python.egg-info/SOURCES.txt

@@ -89,6 +89,7 @@ tests/test_data_input_v2.py
 tests/test_data_timeout_v2.py
 tests/test_datetime_utils.py
 tests/test_range_transform.py
+tests/test_report_delivery.py
 tests/test_report_store.py
 tests/test_request_data_v2.py
 tests/test_result_batch_uploader.py

{pam_python-0.2.19 → pam_python-0.2.20}/tests/test_data_input_v2.py

@@ -47,6 +47,7 @@ class DataInputV2Test(unittest.TestCase):
         self.assertEqual("contacts", data_input.population_mode)
         self.assertEqual(5000, data_input.max_contacts_per_batch)
         self.assertIsNone(data_input.time_window)
+        self.assertIsNone(data_input.missing_contacts)
         self.assertEqual(str(self.population_path.resolve()), data_input.population.path)
         self.assertEqual(1, data_input.population.row_count)
         self.assertEqual(1, len(data_input.datasets))
@@ -56,6 +57,72 @@ class DataInputV2Test(unittest.TestCase):
             data_input.datasets[0].parts[0].path,
         )
 
+    def test_missing_contacts_readers_and_zip_contract(self):
+        import polars as pl
+
+        for file_format in ("csv", "parquet"):
+            for ids in (["contact-2", "contact-3"], []):
+                with self.subTest(format=file_format, ids=ids):
+                    population = self.temp_dir / f"population.{file_format}"
+                    missing = self.temp_dir / f"missing_contacts.{file_format}"
+                    population_frame = pl.DataFrame({"id": ["contact-1"]})
+                    missing_frame = pl.DataFrame({"id": ids}, schema={"id": pl.String})
+                    if file_format == "csv":
+                        population_frame.write_csv(population)
+                        missing_frame.write_csv(missing)
+                    else:
+                        population_frame.write_parquet(population)
+                        missing_frame.write_parquet(missing)
+                    manifest = self._valid_manifest()
+                    manifest["population"]["format"] = file_format
+                    manifest["population"]["file"] = self._metadata(population, 1)
+                    manifest["missing_contacts"] = self._metadata(missing, len(ids))
+                    buffer = BytesIO()
+                    with zipfile.ZipFile(buffer, "w") as archive:
+                        archive.writestr("manifest.json", json.dumps(manifest))
+                        for path in (population, missing, self.dataset_path):
+                            archive.write(path, path.name)
+                    buffer.seek(0)
+                    request, error = self._parse_v2_zip(buffer)
+                    self.assertEqual("", error)
+                    data = request.data_input_v2
+                    self.assertIsNotNone(data)
+                    self.assertIsNotNone(data.missing_contacts)
+                    self.assertEqual(file_format, data.missing_contacts.format)
+                    self.assertEqual(len(ids), data.missing_contacts.row_count)
+                    self.assertEqual(ids, data.missing_contacts.scan_polars().collect()["id"].to_list())
+                    self.assertEqual(ids, data.missing_contacts.read_pandas()["id"].tolist())
+                    if file_format == "csv":
+                        self.assertEqual(ids, [row["id"] for row in data.missing_contacts.iter_csv_rows()])
+                    self.assertEqual(1, len(data.datasets))
+                    missing_path = Path(data.missing_contacts.path)
+                    self.assertTrue(data.cleanup())
+                    self.assertFalse(missing_path.exists())
+
+    def test_missing_contacts_uses_existing_file_validation(self):
+        for failure in ("checksum", "size", "format", "missing", "metadata"):
+            with self.subTest(failure=failure):
+                path = self._write_file("missing_contacts.csv", b"id\ncontact-2\n")
+                manifest = self._valid_manifest()
+                manifest["missing_contacts"] = self._metadata(path, 1)
+                files = [str(self.population_path), str(self.dataset_path), str(path)]
+                if failure == "checksum":
+                    manifest["missing_contacts"]["sha256"] = "0" * 64
+                elif failure == "size":
+                    manifest["missing_contacts"]["size_bytes"] += 1
+                elif failure == "format":
+                    other = path.with_suffix(".parquet")
+                    other.write_bytes(path.read_bytes())
+                    manifest["missing_contacts"] = self._metadata(other, 1)
+                    files[-1] = str(other)
+                elif failure == "missing":
+                    files.pop()
+                else:
+                    manifest["missing_contacts"] = None
+                manifest_path = self._write_manifest(manifest)
+                with self.assertRaises(ValueError):
+                    parse_data_input_v2([str(manifest_path), *files])
+
     def test_pandas_readers_follow_csv_metadata(self):
         manifest_path = self._write_manifest(self._valid_manifest())
         data_input = parse_data_input_v2(

pam_python-0.2.20/tests/test_report_delivery.py

@@ -0,0 +1,34 @@
+import unittest
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+from pam.task_manager import TaskManager
+
+
+class ReportDeliveryTest(unittest.TestCase):
+    def manager_and_service(self):
+        manager = TaskManager.__new__(TaskManager)
+        manager.api = Mock()
+        service = SimpleNamespace(request=SimpleNamespace(
+            service_name="rfm", response_api="https://pam/response", token="secret"))
+        return manager, service
+
+    def test_waits_for_confirmed_enqueue(self):
+        manager, service = self.manager_and_service()
+        def upload(endpoint, path, payload):
+            return Mock(status_code=200, json=lambda: {
+                "report_delivery_id": payload["report_delivery_id"], "stage": "tracker_queued"})
+        manager.api.http_upload.side_effect = upload
+        with patch("pam.task_manager.threading.Thread") as thread:
+            manager.service_upload_report(service, "/tmp/report.csv")
+        thread.assert_not_called()
+        self.assertEqual("secret", manager.api.http_upload.call_args.args[2]["token"])
+
+    def test_failure_and_unconfirmed_ack_raise(self):
+        for response in (None, Mock(status_code=500), Mock(status_code=200, json=lambda: {}),
+                         Mock(status_code=200, json=lambda: {"report_delivery_id": "wrong", "stage": "tracker_queued"})):
+            with self.subTest(response=response):
+                manager, service = self.manager_and_service()
+                manager.api.http_upload.return_value = response
+                with self.assertRaises(RuntimeError):
+                    manager.service_upload_report(service, "/tmp/report.csv")