pam-python 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

pam/__init__.py

@@ -1,3 +1,3 @@
 """PAM Data Plugin framework."""
 
-__version__ = "0.2.2"
+__version__ = "0.2.4"

pam/cli.py

@@ -55,7 +55,7 @@ SERVICE_TEMPLATES = {
 }
 
 ENGINE_DEPENDENCIES = {
-    "polars": ["polars>=1.0.0"],
+    "polars": ["polars>=1.43.0"],
     "pandas": ["pandas>=2.2.3", "pyarrow>=19.0.1"],
 }
 

pam/result_batch_uploader.py

@@ -1,13 +1,29 @@
-"""Bounded, named eager DataFrame batching for plugin result uploads."""
+"""Asynchronous, bounded result batching for PAM CSV uploads."""
 
 from __future__ import annotations
 
 from dataclasses import dataclass
+import queue
 import threading
+import time
 from typing import Any, Dict, Optional, Tuple
 
+from pam.utils import log
+
 
 DEFAULT_BATCH_SIZE = 50000
+DEFAULT_QUEUE_SIZE = 4
+DEFAULT_MAX_RETRIES = 3
+DEFAULT_RETRY_DELAY_SECONDS = 2.0
+DEFAULT_RETRY_MAX_DELAY_SECONDS = 30.0
+_FLUSH = object()
+
+
+@dataclass
+class _ResultInput:
+    dataframe: Any
+    name: str
+    options: Optional[dict]
 
 
 @dataclass
@@ -18,20 +34,56 @@ class _ResultStream:
     buffer: Any
     uploaded_rows: int = 0
     uploaded_batches: int = 0
+    failed_rows: int = 0
+    failed_batches: int = 0
+    retried_uploads: int = 0
 
 
 class ResultBatchUploader:
-    """Accumulate eager Polars or Pandas frames into bounded upload batches.
+    """Queue result frames and upload bounded CSV batches in one worker.
 
-    LazyFrames should be passed directly to ``Service._upload_result`` so Polars
-    can stream the query result to CSV without first collecting every row.
+    Plugin computation and intermediate storage are outside this class. Calls to
+    ``upload`` submit final result frames without waiting for network I/O. A
+    bounded queue applies backpressure when producers outrun the uploader.
+
+    ``flush`` closes input and places a FIFO completion marker. The worker first
+    consumes every earlier result, uploads all complete batches, then uploads all
+    remaining rows. ``wait_for_uploader`` is the completion barrier that must
+    return before a service exits.
     """
 
     def __init__(self, service, batch_size: int = DEFAULT_BATCH_SIZE):
         self._service = service
-        self.batch_size = self._resolve_batch_size(batch_size)
+        self.batch_size = self._resolve_positive_int("batch_size", batch_size)
+        self.queue_size = self._resolve_positive_int(
+            "upload_queue_size", DEFAULT_QUEUE_SIZE
+        )
+        self.max_retries = self._resolve_non_negative_int(
+            "upload_max_retries", DEFAULT_MAX_RETRIES
+        )
+        self.retry_delay_seconds = self._resolve_non_negative_float(
+            "upload_retry_delay_seconds", DEFAULT_RETRY_DELAY_SECONDS
+        )
+        self.retry_max_delay_seconds = self._resolve_non_negative_float(
+            "upload_retry_max_delay_seconds", DEFAULT_RETRY_MAX_DELAY_SECONDS
+        )
+
         self._streams: Dict[str, _ResultStream] = {}
-        self._lock = threading.RLock()
+        self._queue: queue.Queue = queue.Queue(maxsize=self.queue_size)
+        self._submission_lock = threading.Lock()
+        self._state_lock = threading.RLock()
+        self._completion = threading.Event()
+        self._accepting = True
+        self._flush_requested = False
+        self._fatal_error: Optional[BaseException] = None
+        self._state = "open"
+
+        self._worker = threading.Thread(
+            target=self._run_worker,
+            name=f"{self.__class__.__name__}-worker",
+            daemon=True,
+        )
+        self._worker.start()
 
     def upload(
         self,
@@ -39,17 +91,107 @@ class ResultBatchUploader:
         name: str = "default",
         options: Optional[dict] = None,
     ) -> None:
-        engine, normalized = self._normalize_dataframe(dataframe)
+        """Submit a final result without waiting for CSV/network upload."""
         stream_name = self._normalize_name(name)
         normalized_options = self._normalize_options(options)
+        submitted = self._clone_submitted_dataframe(dataframe)
 
-        with self._lock:
-            stream = self._get_or_create_stream(
-                stream_name,
-                engine,
-                normalized,
-                normalized_options,
-            )
+        with self._submission_lock:
+            with self._state_lock:
+                if not self._accepting:
+                    raise RuntimeError("ResultBatchUploader no longer accepts results")
+                if self._fatal_error is not None:
+                    raise RuntimeError("ResultBatchUploader worker failed") from self._fatal_error
+            self._queue.put(_ResultInput(submitted, stream_name, normalized_options))
+
+    def flush(self) -> None:
+        """Close input and enqueue one terminal flush marker without blocking."""
+        with self._submission_lock:
+            with self._state_lock:
+                if self._flush_requested:
+                    return
+                self._accepting = False
+                self._flush_requested = True
+                self._state = "flush_requested"
+            self._queue.put(_FLUSH)
+
+    def wait_for_uploader(self) -> None:
+        """Wait for all queued/retried uploads and the final remainder."""
+        with self._state_lock:
+            if not self._flush_requested:
+                raise RuntimeError("flush() must be called before wait_for_uploader()")
+        self._completion.wait()
+        with self._state_lock:
+            if self._fatal_error is not None:
+                raise RuntimeError("ResultBatchUploader worker failed") from self._fatal_error
+
+    def get_status(self) -> Dict[str, dict]:
+        with self._state_lock:
+            return {
+                name: {
+                    "buffered_rows": len(stream.buffer),
+                    "uploaded_rows": stream.uploaded_rows,
+                    "uploaded_batches": stream.uploaded_batches,
+                    "failed_rows": stream.failed_rows,
+                    "failed_batches": stream.failed_batches,
+                    "retried_uploads": stream.retried_uploads,
+                }
+                for name, stream in self._streams.items()
+            }
+
+    @property
+    def state(self) -> str:
+        with self._state_lock:
+            return self._state
+
+    def _run_worker(self) -> None:
+        try:
+            while True:
+                item = self._queue.get()
+                try:
+                    if item is _FLUSH:
+                        if self._fatal_error is None:
+                            self._flush_all_streams()
+                        with self._state_lock:
+                            self._state = (
+                                "failed" if self._fatal_error is not None else "completed"
+                            )
+                        return
+                    if self._fatal_error is None:
+                        self._consume_input(item)
+                except BaseException as exc:  # keep worker alive until the flush barrier
+                    self._record_fatal_error(exc)
+                finally:
+                    self._queue.task_done()
+        finally:
+            with self._state_lock:
+                self._accepting = False
+                if self._state not in {"completed", "failed"}:
+                    self._state = "failed"
+            self._completion.set()
+
+    def _consume_input(self, item: _ResultInput) -> None:
+        dataframe = item.dataframe
+        if self._is_polars_lazyframe(dataframe):
+            for batch in dataframe.collect_batches(
+                chunk_size=self.batch_size,
+                maintain_order=True,
+                engine="streaming",
+            ):
+                self._consume_eager(batch, item.name, item.options)
+            return
+        self._consume_eager(dataframe, item.name, item.options)
+
+    def _consume_eager(
+        self,
+        dataframe: Any,
+        name: str,
+        options: Optional[dict],
+    ) -> None:
+        engine, normalized = self._normalize_eager_dataframe(dataframe)
+        batches = []
+        with self._state_lock:
+            stream = self._get_or_create_stream(name, engine, normalized, options)
             if len(normalized) == 0:
                 return
 
@@ -68,12 +210,11 @@ class ResultBatchUploader:
                     len(remaining) - rows_needed,
                 )
                 if len(stream.buffer) == self.batch_size:
-                    self._upload_batch(stream, stream.buffer)
+                    batches.append(self._clone(engine, stream.buffer))
                     stream.buffer = self._empty_frame(engine, stream.buffer)
 
             while len(remaining) >= self.batch_size:
-                batch = self._slice(engine, remaining, 0, self.batch_size)
-                self._upload_batch(stream, batch)
+                batches.append(self._slice(engine, remaining, 0, self.batch_size))
                 remaining = self._slice(
                     engine,
                     remaining,
@@ -84,30 +225,64 @@ class ResultBatchUploader:
             if len(remaining) > 0:
                 stream.buffer = self._clone(engine, remaining)
 
-    def flush(self, name: Optional[str] = None) -> None:
-        with self._lock:
-            stream_names = (
-                list(self._streams)
-                if name is None
-                else [self._normalize_name(name)]
-            )
-            for stream_name in stream_names:
-                stream = self._streams.get(stream_name)
-                if stream is None or len(stream.buffer) == 0:
+        for batch in batches:
+            self._upload_batch(name, stream, batch)
+
+    def _flush_all_streams(self) -> None:
+        pending = []
+        with self._state_lock:
+            for name, stream in self._streams.items():
+                if len(stream.buffer) == 0:
                     continue
-                self._upload_batch(stream, stream.buffer)
+                pending.append((name, stream, self._clone(stream.engine, stream.buffer)))
                 stream.buffer = self._empty_frame(stream.engine, stream.buffer)
+        for name, stream, batch in pending:
+            self._upload_batch(name, stream, batch)
 
-    def get_status(self) -> Dict[str, dict]:
-        with self._lock:
-            return {
-                name: {
-                    "buffered_rows": len(stream.buffer),
-                    "uploaded_rows": stream.uploaded_rows,
-                    "uploaded_batches": stream.uploaded_batches,
-                }
-                for name, stream in self._streams.items()
-            }
+    def _upload_batch(self, name: str, stream: _ResultStream, dataframe: Any) -> None:
+        attempts = self.max_retries + 1
+        for attempt in range(1, attempts + 1):
+            try:
+                self._service._upload_result(
+                    self._clone(stream.engine, dataframe),
+                    stream.options,
+                )
+                with self._state_lock:
+                    stream.uploaded_rows += len(dataframe)
+                    stream.uploaded_batches += 1
+                return
+            except Exception as exc:  # network/upload failures are retryable
+                if attempt < attempts:
+                    with self._state_lock:
+                        stream.retried_uploads += 1
+                    delay = min(
+                        self.retry_delay_seconds * (2 ** (attempt - 1)),
+                        self.retry_max_delay_seconds,
+                    )
+                    log(
+                        f"Result upload retry stream={name} rows={len(dataframe)} "
+                        f"attempt={attempt + 1}/{attempts} delay={delay}s error={exc}",
+                        level="WARNING",
+                    )
+                    if delay > 0:
+                        time.sleep(delay)
+                    continue
+
+                with self._state_lock:
+                    stream.failed_rows += len(dataframe)
+                    stream.failed_batches += 1
+                log(
+                    f"Result upload skipped after retries stream={name} "
+                    f"rows={len(dataframe)} attempts={attempts} error={exc}",
+                    level="ERROR",
+                )
+
+    def _record_fatal_error(self, exc: BaseException) -> None:
+        with self._state_lock:
+            if self._fatal_error is None:
+                self._fatal_error = exc
+                self._accepting = False
+        log(f"Result uploader worker failed: {exc}", level="ERROR")
 
     def _get_or_create_stream(
         self,
@@ -138,46 +313,56 @@ class ResultBatchUploader:
             raise ValueError(f"Result stream '{name}' upload options changed")
         return stream
 
-    def _upload_batch(self, stream: _ResultStream, dataframe: Any) -> None:
-        self._service._upload_result(
-            self._clone(stream.engine, dataframe),
-            stream.options,
-        )
-        stream.uploaded_rows += len(dataframe)
-        stream.uploaded_batches += 1
-
-    def _resolve_batch_size(self, configured_default: int) -> int:
-        fallback = (
-            configured_default
-            if isinstance(configured_default, int)
-            and not isinstance(configured_default, bool)
-            and configured_default > 0
-            else DEFAULT_BATCH_SIZE
-        )
-        raw_value = getattr(self._service.request, "runtime_parameters", {}).get(
-            "batch_size"
-        )
-        if raw_value is None:
+    def _runtime_parameters(self) -> dict:
+        return getattr(self._service.request, "runtime_parameters", {})
+
+    def _resolve_positive_int(self, key: str, fallback: int) -> int:
+        try:
+            value = int(self._runtime_parameters().get(key, fallback))
+        except (TypeError, ValueError):
             return fallback
+        return value if value > 0 else fallback
+
+    def _resolve_non_negative_int(self, key: str, fallback: int) -> int:
         try:
-            parsed = int(raw_value)
+            value = int(self._runtime_parameters().get(key, fallback))
         except (TypeError, ValueError):
             return fallback
-        return parsed if parsed > 0 else fallback
+        return value if value >= 0 else fallback
+
+    def _resolve_non_negative_float(self, key: str, fallback: float) -> float:
+        try:
+            value = float(self._runtime_parameters().get(key, fallback))
+        except (TypeError, ValueError):
+            return fallback
+        return value if value >= 0 else fallback
+
+    @classmethod
+    def _clone_submitted_dataframe(cls, dataframe: Any):
+        if cls._is_polars_lazyframe(dataframe):
+            return dataframe.clone()
+        _, normalized = cls._normalize_eager_dataframe(dataframe)
+        return normalized
 
     @staticmethod
-    def _normalize_dataframe(dataframe: Any) -> tuple[str, Any]:
+    def _normalize_eager_dataframe(dataframe: Any) -> tuple[str, Any]:
         module_name = type(dataframe).__module__.split(".", 1)[0]
         class_name = type(dataframe).__name__
-        if module_name == "polars" and class_name == "LazyFrame":
-            raise TypeError(
-                "Polars LazyFrame must be uploaded directly with Service._upload_result"
-            )
         if module_name == "polars" and class_name == "DataFrame":
             return "polars", dataframe.clone()
         if module_name == "pandas" and class_name == "DataFrame":
             return "pandas", dataframe.reset_index(drop=True).copy()
-        raise TypeError("dataframe must be a Polars or Pandas DataFrame")
+        raise TypeError(
+            "dataframe must be a Polars LazyFrame, Polars DataFrame, "
+            "or Pandas DataFrame"
+        )
+
+    @staticmethod
+    def _is_polars_lazyframe(dataframe: Any) -> bool:
+        return (
+            type(dataframe).__module__.split(".", 1)[0] == "polars"
+            and type(dataframe).__name__ == "LazyFrame"
+        )
 
     @staticmethod
     def _clone(engine: str, dataframe: Any):

pam/task_manager.py

@@ -305,7 +305,10 @@ class TaskManager(ITaskManager):
 
     def service_upload_result(self, service: Service, file_path, options=None):
         """
-        Uploads a result file asynchronously and logs the response.
+        Uploads one result file synchronously.
+
+        ResultBatchUploader owns background concurrency and invokes this method
+        from its worker, so failures must propagate for retry handling.
         """
         endpoint = service.request.response_api
         payload = None
@@ -318,25 +321,21 @@ class TaskManager(ITaskManager):
                 is_priority = options.get("is_priority")
                 payload["is_priority"] = str(is_priority).lower() if isinstance(is_priority, bool) else is_priority
 
-        def handle_upload_response(response):
-            """Logs the response after the upload completes."""
-            if response is None:
-                log(f"Response from upload to {endpoint}: None")
-                return
-            try:
-                response_data = response.json()
-            except ValueError:
-                response_data = response.text
-            log(f"Response from upload to {endpoint}: {response_data}")
-
-        def upload_wrapper():
-            """Wrapper for the upload to handle response logging."""
-            response = self.api.http_upload(endpoint, file_path, payload)
-            handle_upload_response(response)
-
         log(f"Uploading Result to: {endpoint}")
-        http_thread = threading.Thread(target=upload_wrapper)
-        http_thread.start()
+        response = self.api.http_upload(endpoint, file_path, payload)
+        if response is None:
+            raise RuntimeError(f"No response from result upload to {endpoint}")
+        if not response.ok:
+            raise RuntimeError(
+                f"Result upload failed status={response.status_code} "
+                f"endpoint={endpoint}"
+            )
+        try:
+            response_data = response.json()
+        except ValueError:
+            response_data = response.text
+        log(f"Response from upload to {endpoint}: {response_data}")
+        return response
 
     def service_upload_report(self, service: Service, file_path):
         """

pam/templates/init/AGENT.md

@@ -117,15 +117,17 @@ input data. CDP returns up to 5 Parquet files (each file is a separate event).
 
 - File order is guaranteed per agreement, so you can index by position.
 - Column names and record counts are defined by CDP configuration.
+- `input_files` is not a list of interchangeable files. Each position represents
+  a different configured data source and must be bound to a meaningful domain name.
+- Never process `input_files` with `for file_path in req.input_files`.
 
 Example file order usage:
 
 ```python
-input_files = req.input_files
-# Index 0: purchase events
-# Index 1: use points events
-purchase_path = input_files[0]
-use_points_path = input_files[1]
+if len(req.input_files) != 2:
+    raise ValueError("Expected purchase_event and point_received_event inputs")
+
+purchase_event_file, point_received_event_file = req.input_files
 ```
 
 Column meaning
@@ -148,6 +150,11 @@ When CDP finishes collecting data, it calls `on_data_input`.
 Example:
 
 ```python
+def __init__(self, task_manager, req):
+    super().__init__(task_manager, req)
+    self.batch_uploader = ResultBatchUploader(self, batch_size=50000)
+
+
 def on_data_input(self, req: RequestCommand):
     log(f"on_data_input req.is_end = {req.is_end}")
 
@@ -163,26 +170,40 @@ def __run_process_data_in_thread(self, req: RequestCommand):
     if req.file_format is not RequestFileFormat.PARQUET:
         raise ValueError(f"Expected parquet, received {req.file_format.value}")
 
-    result = transform(pl.scan_parquet(req.input_files[0]))
-    self._upload_result(result)
+    if len(req.input_files) != 2:
+        raise ValueError("Expected purchase_event and point_received_event inputs")
+
+    purchase_event_file, point_received_event_file = req.input_files
+    result = transform(
+        pl.scan_parquet(purchase_event_file),
+        pl.scan_parquet(point_received_event_file),
+    )
+    self.batch_uploader.upload(result, name="main")
 
     if not req.is_end:
-        self._request_data(req.next, file_format=RequestFileFormat.PARQUET)
+        self._request_data(
+            page=req.next,
+            file_format=RequestFileFormat.PARQUET,
+        )
     else:
+        self.batch_uploader.flush()
+        self.batch_uploader.wait_for_uploader()
         self._exit()
 ```
 
 Why a thread: to end the HTTP request quickly and avoid timeouts.
 
-`req.input_files` is the ordered list of Parquet paths. `req.file_format` is the
-typed metadata sent by PAM and is the source of truth; validate file extensions as
-an additional consistency check.
+`req.input_files` is the ordered positional contract of Parquet paths.
+`req.file_format` is the typed metadata sent by PAM and is the source of truth;
+validate file extensions as an additional consistency check.
 
 ---
 
 ## Upload Results to CDP
 
-After processing, call `_upload_result` with a dataframe. Example output:
+After processing, submit final dataframe results through `ResultBatchUploader`.
+The uploader creates bounded CSV files and sends them through the PAM API.
+Example output:
 
 ```csv
 id,data_x,rfm
@@ -258,21 +279,32 @@ CDP paging is **customer-based**, not row-based.
 
 When requesting data page by page:
 
-- CDP collects customers in batches of approximately **10,000 customers per page**.
-- All events belonging to those customers are merged into **a single Parquet file**.
-- The number of rows is **unknown** and depends on how many events each customer has.
+- PAM first selects a cohort of approximately **10,000 unique customers per page**.
+- PAM then queries **every configured dataset** with that same customer-ID cohort.
+- Dataset 1 through dataset N in one callback therefore belong to the same group
+  of customers, while retaining their configured positional meanings.
+- Each dataset has its own Parquet file and may have a different row count because
+  event density differs and some customers may have no rows in a dataset.
+- A customer's relevant data is kept within one logical page instead of being
+  split across customer pages.
+- `req.next` advances the customer-cohort cursor, not an event-row offset.
 
 This means:
 
 - Each page represents a logical customer group, not a fixed number of records.
 - A page may contain a small or very large number of rows depending on behavior density.
+- Files at different input positions must be joined or compared by the agreed
+  customer key; row positions across files do not correspond.
 
 ### Execution Implications
 
 Because paging is based on customers:
 
 - In many cases, a service can process **one page (≈10,000 customers)** independently.
-- If the logic allows, you may process and upload results immediately because the dataset is self-contained.
+- If the logic allows, you may process and upload results immediately because all
+  configured datasets for that customer cohort arrive together.
+- Do not create row-offset pagination or carry partial state because one dataset
+  happens to have more event rows than another.
 
 However, upload behavior must balance two constraints:
 
@@ -368,6 +400,11 @@ Report types outside that closed contract require a newer framework and CMS rele
 When uploading large DataFrames, always upload in batches to avoid memory/network spikes.
 Batch size MUST be configurable via `request.runtime_parameters["batch_size"]`.
 
+Computation is a developer-owned black box. The service may calculate within one
+page, retain state across pages, or use any intermediate store such as SQLite or
+DuckDB. The framework does not choose that strategy. Call the uploader only when
+the produced rows are ready for the PAM result contract.
+
 ## Required behavior
 
 - Read batch size from `self.request.runtime_parameters.get("batch_size", "50000")`
@@ -379,10 +416,17 @@ Batch size MUST be configurable via `request.runtime_parameters["batch_size"]`.
 
 ## Reference implementation pattern
 
-For a Polars `LazyFrame`, call `_upload_result(lazy_frame)` directly so the runtime
-can stream it to CSV. For eager Polars/Pandas frames, `ResultBatchUploader` reads
-`runtime_parameters["batch_size"]`, keeps named result streams separate, uploads
-complete batches, and flushes remaining rows when successful processing finishes.
+`ResultBatchUploader` accepts Polars `LazyFrame`, Polars `DataFrame`, and Pandas
+`DataFrame`. It evaluates lazy results as bounded streaming chunks, reads
+`runtime_parameters["batch_size"]`, keeps named result streams separate, converts
+complete batches to CSV, and uploads through the PAM API. Incomplete batches may
+remain buffered across pages or calls until more final rows arrive or `flush()` is
+called.
+
+`upload()` queues work for one background uploader and normally returns without
+waiting for network I/O, allowing the service to request and process the next PAM
+page. The queue is bounded, so `upload()` may apply backpressure if uploads are
+slower than result production.
 
 ```python
 from pam.result_batch_uploader import ResultBatchUploader
@@ -390,13 +434,36 @@ from pam.result_batch_uploader import ResultBatchUploader
 self.batch_uploader = ResultBatchUploader(self, batch_size=50000)
 self.batch_uploader.upload(result_df, name="main")
 self.batch_uploader.flush()
+self.batch_uploader.wait_for_uploader()
+self._exit()
 ```
 
+`flush()` is both a terminal state change and a FIFO marker. It stops new result
+submissions but does not wait. The worker processes every result submitted before
+the marker, preserves `batch_size`, uploads the final remainder, and then completes.
+`wait_for_uploader()` blocks until queued and in-flight network uploads finish.
+It intentionally has no timeout: timeout each HTTP upload attempt instead, then
+retry within the configured limit or skip the permanently failed batch. A timeout
+on the completion barrier could let the plugin exit while uploads are still active.
+
+Upload retry settings are optional runtime parameters:
+
+- `upload_queue_size` (default `4`): maximum queued result objects before backpressure
+- `upload_max_retries` (default `3`): retries after the first upload attempt
+- `upload_retry_delay_seconds` (default `2`): initial exponential-backoff delay
+- `upload_retry_max_delay_seconds` (default `30`): backoff delay cap
+
+After retries are exhausted, the uploader logs the failed batch, records
+`failed_rows`/`failed_batches`, skips it, and continues with the next batch.
+`wait_for_uploader()` still waits for the whole queue before returning.
+
 ## Notes
 
 - Do NOT hardcode a fixed batch size inside upload loops.
 - Do NOT read environment variables for batch sizing unless explicitly required; use `runtime_parameters` only.
-- Do NOT pass a Polars `LazyFrame` to `ResultBatchUploader`.
+- Do NOT call `_upload_result(result)` directly from plugin code; use `ResultBatchUploader`.
+- Call `flush()` only after all intended final rows have been submitted.
+- Always call `wait_for_uploader()` after `flush()` and before `_exit()`.
 
 ---
 
@@ -419,7 +486,7 @@ Prefer streaming/chunked/incremental processing whenever possible.
 
 - Start with `pl.scan_parquet(...)`, not `pl.read_parquet(...)`.
 - Select required columns and filter rows early for projection/predicate pushdown.
-- Keep transformations as a `LazyFrame` through `_upload_result(...)`.
+- Keep transformations as a `LazyFrame` through `ResultBatchUploader.upload(...)`.
 
 ### 2. DB-first computation
 

pam/templates/service/pandas/service.test.tmpl

@@ -18,6 +18,7 @@ from #MODULE_NAME#.#CLASS_NAME# import #CLASS_NAME#
 TOTAL_PAGES = 3
 MOCK_DIR = "./TEMP_TEST_DATA"
 UPLOADED_RESULTS = []
+UPLOADED_ROW_COUNTS = []
 REQUESTED_FORMATS = []
 TempfileUtils.temp_base_path = f"{MOCK_DIR}/app/data"
 TempfileUtils.temp_datasource_path = f"{MOCK_DIR}/app/data/data_sources"
@@ -68,6 +69,7 @@ def on_publish_report_pointer(report_key, pointer):
 
 def on_upload_result(result_file):
     UPLOADED_RESULTS.append(result_file)
+    UPLOADED_ROW_COUNTS.append(len(pd.read_csv(result_file)))
 
 
 class Test#CLASS_NAME#(unittest.TestCase):
@@ -77,6 +79,7 @@ class Test#CLASS_NAME#(unittest.TestCase):
         os.makedirs(MOCK_DIR)
         create_mock_parquet(TOTAL_PAGES, MOCK_DIR)
         UPLOADED_RESULTS.clear()
+        UPLOADED_ROW_COUNTS.clear()
         REQUESTED_FORMATS.clear()
 
         self.tester_task = TesterTask()
@@ -112,6 +115,7 @@ class Test#CLASS_NAME#(unittest.TestCase):
 
         self.assertTrue(self.tester_task.is_exit)
         self.assertEqual(2, len(UPLOADED_RESULTS))
+        self.assertEqual([15, 15], UPLOADED_ROW_COUNTS)
         self.assertEqual(
             [RequestFileFormat.PARQUET] * TOTAL_PAGES,
             REQUESTED_FORMATS,

pam/templates/service/pandas/service_class.tmpl

@@ -13,6 +13,7 @@ from #MODULE_NAME#.functions import load_input_file, transform
 
 class #CLASS_NAME#(Service):
     INPUT_FILE_FORMAT = RequestFileFormat.PARQUET
+    EXPECTED_DATASET_COUNT = 1
 
     def __init__(self, task_manager, req):
         super().__init__(task_manager, req)
@@ -40,11 +41,22 @@ class #CLASS_NAME#(Service):
                 f"received {req.file_format.value}"
             )
 
-        for file_path in req.input_files:
-            if not file_path.lower().endswith(".parquet"):
-                raise ValueError(f"Expected a .parquet input file: {file_path}")
-            result = transform(load_input_file(file_path))
-            self.batch_uploader.upload(result, name="main")
+        if len(req.input_files) != self.EXPECTED_DATASET_COUNT:
+            raise ValueError(
+                f"Expected {self.EXPECTED_DATASET_COUNT} configured dataset, "
+                f"received {len(req.input_files)}"
+            )
+
+        # Bind every position to the data source name configured in PAM.
+        # Add more named positions here when the service contract has more datasets.
+        (input_dataset_file,) = req.input_files
+        if not input_dataset_file.lower().endswith(".parquet"):
+            raise ValueError(
+                f"Expected a .parquet input file: {input_dataset_file}"
+            )
+
+        result = transform(load_input_file(input_dataset_file))
+        self.batch_uploader.upload(result, name="main")
 
         if not req.is_end:
             self._request_data(
@@ -53,6 +65,7 @@ class #CLASS_NAME#(Service):
             )
         else:
             self.batch_uploader.flush()
+            self.batch_uploader.wait_for_uploader()
             self._exit()
 
     def on_destroy(self):

pam/templates/service/polars/service.test.tmpl

@@ -18,6 +18,7 @@ from #MODULE_NAME#.#CLASS_NAME# import #CLASS_NAME#
 TOTAL_PAGES = 3
 MOCK_DIR = "./TEMP_TEST_DATA"
 UPLOADED_RESULTS = []
+UPLOADED_ROW_COUNTS = []
 REQUESTED_FORMATS = []
 TempfileUtils.temp_base_path = f"{MOCK_DIR}/app/data"
 TempfileUtils.temp_datasource_path = f"{MOCK_DIR}/app/data/data_sources"
@@ -68,6 +69,7 @@ def on_publish_report_pointer(report_key, pointer):
 
 def on_upload_result(result_file):
     UPLOADED_RESULTS.append(result_file)
+    UPLOADED_ROW_COUNTS.append(pl.read_csv(result_file).height)
 
 
 class Test#CLASS_NAME#(unittest.TestCase):
@@ -77,6 +79,7 @@ class Test#CLASS_NAME#(unittest.TestCase):
         os.makedirs(MOCK_DIR)
         create_mock_parquet(TOTAL_PAGES, MOCK_DIR)
         UPLOADED_RESULTS.clear()
+        UPLOADED_ROW_COUNTS.clear()
         REQUESTED_FORMATS.clear()
 
         self.tester_task = TesterTask()
@@ -96,7 +99,7 @@ class Test#CLASS_NAME#(unittest.TestCase):
             sqlite_download="",
             sqlite_download_v2="",
             sqlite_upload="",
-            runtime_parameters={"some_param": "some_value"},
+            runtime_parameters={"some_param": "some_value", "batch_size": "15"},
             token=uuid.uuid4().hex,
             cmd="start",
             data_api="",
@@ -111,7 +114,8 @@ class Test#CLASS_NAME#(unittest.TestCase):
         self.tester_task.wait_for_task_done()
 
         self.assertTrue(self.tester_task.is_exit)
-        self.assertEqual(TOTAL_PAGES, len(UPLOADED_RESULTS))
+        self.assertEqual(2, len(UPLOADED_RESULTS))
+        self.assertEqual([15, 15], UPLOADED_ROW_COUNTS)
         self.assertEqual(
             [RequestFileFormat.PARQUET] * TOTAL_PAGES,
             REQUESTED_FORMATS,

pam/templates/service/polars/service_class.tmpl

@@ -3,6 +3,7 @@ import threading
 
 from pam.models.request_command import RequestCommand
 from pam.request_file_format import RequestFileFormat
+from pam.result_batch_uploader import ResultBatchUploader
 from pam.service import Service
 from pam.utils import log
 
@@ -12,6 +13,11 @@ from #MODULE_NAME#.functions import load_input_file, transform
 
 class #CLASS_NAME#(Service):
     INPUT_FILE_FORMAT = RequestFileFormat.PARQUET
+    EXPECTED_DATASET_COUNT = 1
+
+    def __init__(self, task_manager, req):
+        super().__init__(task_manager, req)
+        self.batch_uploader = ResultBatchUploader(self, batch_size=50000)
 
     def on_start(self):
         log("on_start")
@@ -35,11 +41,22 @@ class #CLASS_NAME#(Service):
                 f"received {req.file_format.value}"
             )
 
-        for file_path in req.input_files:
-            if not file_path.lower().endswith(".parquet"):
-                raise ValueError(f"Expected a .parquet input file: {file_path}")
-            result = transform(load_input_file(file_path))
-            self._upload_result(result)
+        if len(req.input_files) != self.EXPECTED_DATASET_COUNT:
+            raise ValueError(
+                f"Expected {self.EXPECTED_DATASET_COUNT} configured dataset, "
+                f"received {len(req.input_files)}"
+            )
+
+        # Bind every position to the data source name configured in PAM.
+        # Add more named positions here when the service contract has more datasets.
+        (input_dataset_file,) = req.input_files
+        if not input_dataset_file.lower().endswith(".parquet"):
+            raise ValueError(
+                f"Expected a .parquet input file: {input_dataset_file}"
+            )
+
+        result = transform(load_input_file(input_dataset_file))
+        self.batch_uploader.upload(result, name="main")
 
         if not req.is_end:
             self._request_data(
@@ -47,6 +64,8 @@ class #CLASS_NAME#(Service):
                 file_format=self.INPUT_FILE_FORMAT,
             )
         else:
+            self.batch_uploader.flush()
+            self.batch_uploader.wait_for_uploader()
             self._exit()
 
     def on_destroy(self):
@@ -56,4 +75,4 @@ class #CLASS_NAME#(Service):
         log("on_terminate")
 
     def get_status(self):
-        return json.dumps({"engine": "polars"})
+        return json.dumps(self.batch_uploader.get_status(), sort_keys=True)

pam/tester_task.py

@@ -207,12 +207,10 @@ class TesterTask(ITaskManager):
         :param options: Optional upload options (e.g. is_realtime, is_priority).
         """
         if self.upload_result_callback is not None:
-            try:
-                self.upload_result_callback(file_path)
-            except Exception as e:
-                print(f"Error in upload_result_callback: {e}")
+            return self.upload_result_callback(file_path)
         else:
             print("Upload result callback is not set.")
+        return None
 
     def service_upload_report(self, service: Service, file_path: str):
         """

{pam_python-0.2.2.dist-info → pam_python-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.2.2
+Version: 0.2.4
 Summary: Pam Python Library
 Author-email: Narongrit Kanhanoi <narongrit@pams.ai>
 Project-URL: Homepage, https://github.com/heart/pam-python
@@ -122,13 +122,13 @@ The runtime calls your service in two main phases.
 2. `on_data_input`
 
 - Called when CDP sends input files
-- `req.input_files` contains ordered Parquet files
+- `req.input_files` is the ordered positional data-source contract configured in PAM
 - `req.file_format` is the typed format sent by PAM
 - Should also return quickly (use a thread if needed)
 
 When your service is done:
 
-- Upload ordinary CDP result rows through `ResultBatchUploader` or `_upload_result(...)`
+- Upload ordinary CDP result rows through `ResultBatchUploader`
 - Publish managed reports through `self.reports`; do not build report JSON manually
 - Call `self._exit()` to signal completion
 
@@ -151,11 +151,11 @@ Notes:
 ---
 
 **Uploading Results in Batches**
-Polars templates pass `LazyFrame` results directly to `_upload_result(...)`, which
-streams them to the CSV upload boundary. Do not collect or convert to Pandas first.
-
-For eager Polars or Pandas DataFrames, use the batch uploader to handle chunking
-and flushing automatically.
+Polars `LazyFrame`/`DataFrame` and Pandas `DataFrame` results all pass through
+`ResultBatchUploader`. It streams lazy results in bounded chunks, buffers incomplete
+batches across calls, writes complete batches as CSV, and uploads them through PAM.
+The framework does not prescribe page-local/global computation or intermediate
+storage; submit rows when your business result is ready.
 
 Recommended usage:
 
@@ -165,6 +165,7 @@ from pam.result_batch_uploader import ResultBatchUploader
 batch_uploader = ResultBatchUploader(self, batch_size=50000)
 batch_uploader.upload(df, name="main")
 batch_uploader.flush()
+batch_uploader.wait_for_uploader()
 status = batch_uploader.get_status()
 ```
 
@@ -173,9 +174,12 @@ Notes:
 - `request.runtime_parameters["batch_size"]` overrides the constructor default when valid.
 - `name` separates result streams and enforces one stable column schema per stream.
 - `options` may be passed to `upload(...)` and are forwarded to `_upload_result`.
-- Complete batches upload immediately; `flush()` sends the remaining rows.
-- `get_status()` returns buffered rows, uploaded rows, and uploaded batch counts per stream.
-- `ResultBatchUploader` intentionally rejects Polars `LazyFrame`; upload it directly.
+- Uploads run in a bounded background queue so processing can continue while PAM receives CSV batches.
+- `flush()` closes input and queues the final remainder; it does not wait for network completion.
+- Always call `wait_for_uploader()` after `flush()` and before `_exit()`.
+- Failed uploads retry with exponential backoff, then log/record the failed batch and continue.
+- `get_status()` includes buffered, uploaded, failed, and retried counts per stream.
+- Polars `LazyFrame` is supported without collecting the complete result at once.
 
 ---
 
@@ -304,6 +308,5 @@ After `pam init` and one service:
 - Implement your logic in `functions.py`.
 - Wire it into `on_start` and `on_data_input` in your service class.
 - Use the temp utilities to write intermediate files.
-- Keep Polars transformations lazy and upload the resulting `LazyFrame` directly.
-- Use `ResultBatchUploader` only for eager Polars/Pandas frames that need row batching.
+- Keep Polars transformations lazy and submit final results through `ResultBatchUploader`.
 - Read `REPORTS.md` before implementing a managed report.

{pam_python-0.2.2.dist-info → pam_python-0.2.4.dist-info}/RECORD

@@ -1,16 +1,16 @@
-pam/__init__.py,sha256=vwt2aChyIDuFICQ8ZWjM1dPatwHoyi1kwsjGs8LoLhE,56
+pam/__init__.py,sha256=cSLmJntln2OVNNJ07_9Z2eKFLloAZc33rsRlnRTj0wk,56
 pam/api.py,sha256=4t64ey_Jc4gC3SvfI-mtedEGzkX2cPTt0cdZgP_huLk,7231
-pam/cli.py,sha256=GbTyHD2Gc2WLTY54bZ9rtofjVc7bV2JjqXThR9JCQf4,14130
+pam/cli.py,sha256=Kbl5K4xS5KtW1cU3QRZOi8eAdKzUaJqe2JDGuhy4wQA,14131
 pam/interface_task_manager.py,sha256=moKUjSpAeNF9C2dluDFPAdU4IyvwXckB0RK-jINTLm4,1665
 pam/logger.py,sha256=JYPfIpaQZy6QrfVKrzEMrxOgTzWIEhoegvIZgHTGWoc,3021
 pam/request_file_format.py,sha256=fSRlJagbvVGqdIEqhINSeqBNUTNQdm8GL5JfqgX9bxU,317
-pam/result_batch_uploader.py,sha256=RVnaE-Itfacx1yoBJfhWwqaXk7ytEKyvLM3ZNlj6Pug,7997
+pam/result_batch_uploader.py,sha256=xw4b-q7i52CoCjvQtQ8EgfBrrqAl1zEl6sTO-wIvFPc,15424
 pam/server.py,sha256=ZRukc3uLJXxJdWIQDcPqrxjd_LnsGTxPJQtLX0bMcRc,7594
 pam/service.py,sha256=EymzqpHwjmCpSC3e3QchdnrE7L39_hxtDVB3kCJkfr4,7760
 pam/sqlite.py,sha256=0bEUnb17V_Z6N6u843MYyGQnigaAuMx6FUckeWifn8I,2585
-pam/task_manager.py,sha256=tgxf0pkK8zn-BcPNHvzkgHAZ4oNGuhoCyqL4Q69R9dY,15839
+pam/task_manager.py,sha256=DC2pZ1iJU-GnCl8mawuC6Ym105ywoZ6rs6neXUexHZ4,15797
 pam/temp_file_utils.py,sha256=lVkRJuNMdmqbl22BeP4tGo2YAdYy4C_SMzcR4vSQOb4,6762
-pam/tester_task.py,sha256=RhYCh9ia_-bkYdHjRkZmFyJectmW8wcOr1EjPHWnJ9o,9305
+pam/tester_task.py,sha256=s6BelOUMOahUh_5-lRN5yUEO72caR-ERQS8b3ZpfQgw,9213
 pam/utils.py,sha256=-pfnQ00Ib-IXvrMz4S5lTlO36PtSaJoQOhwXpwet8h4,1744
 pam/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pam/models/request_command.py,sha256=tGzPuP1TQI0CrNjiigW929lJ99Fvz-WfcUh02eOVFK4,6621
@@ -48,7 +48,7 @@ pam/reports/transport.py,sha256=t3oEVptS9msnNafia1G8vmTUCdhxvV5L2-0AY-8SKFk,2062
 pam/reports/treemap.py,sha256=W_VwxbiRMSvkNpY2z1HfbZtGKWXNr6FdbVdI0grVesw,4251
 pam/templates/buildcmd/pamb,sha256=58GCN59ysvHGNhPvTD3nkF4aLg-kM1WfWmH7CL3eQVo,530
 pam/templates/buildcmd/pamb-base.sh,sha256=Q0DkbeHtPx33PIjc68EeI7ZIwBOLlwcSmPhpxALXxVY,518
-pam/templates/init/AGENT.md,sha256=XoguXdHm7qw4ssHIL7yP3ZVhmiPosOc0Js_WCLbw_3o,14822
+pam/templates/init/AGENT.md,sha256=uEvHox6K43Em2vWQ8H_5a2NswLUdlSmfve3cWEwLDes,18524
 pam/templates/init/REPORTS.md,sha256=rg9oi0_xDYr0BZBluJh0vPF8lSx7mCRFpDCpUgNl68A,42851
 pam/templates/init/dockerignore.tmpl,sha256=Ht1eCgQLqalEwpUjih0FQavzhYJbT2jH-FP4btM3960,52
 pam/templates/init/gitignore.tmpl,sha256=U59ynPRu_Kw6evqQzOOTAhWKdIXbf1uBOF2V8223tU8,3510
@@ -64,14 +64,14 @@ pam/templates/project/uv/pyproject.toml,sha256=30a2hJn6oN1AGvmgEBdxLETOBlHY8L-II
 pam/templates/project/uv/python-version,sha256=e1X45ntWI8S-8_ppEojalDfXnTq6FW3kjUgdsyrH0W0,5
 pam/templates/service/common/service.yaml,sha256=ceyFswXjTh07vB4hrJEu7pj3mjlq3NPD4GPYTmvGwmE,44
 pam/templates/service/pandas/functions.tmpl,sha256=ziHgPaK7i8ESY-TxPMuKbqAFILjzdgn1p5FdPf3pYuw,360
-pam/templates/service/pandas/service.test.tmpl,sha256=PybHu_c5BMhUmvkc9ycQH7vJbPWt7isJAH2r-ElQ9dE,3760
-pam/templates/service/pandas/service_class.tmpl,sha256=LGgIYjfdsBwppOqP-FfgEsF1kxeSwUnYwK935gqC7GQ,2138
+pam/templates/service/pandas/service.test.tmpl,sha256=kK86qW79eFtJCGSqUe0p0dckQlo9DnGzAcTkGB97WKo,3939
+pam/templates/service/pandas/service_class.tmpl,sha256=Pzw2Wcv1FZMDbWAcfakM2V0QhxKjq31mX5bfv5ekeTI,2666
 pam/templates/service/polars/functions.tmpl,sha256=nMKFC5GqIm2Q1dSg2zDOscSwQaOpzS2jzqjQV7xhPRk,382
-pam/templates/service/polars/service.test.tmpl,sha256=58x_MJYwp7BvDtuSsETAt9Q6v5KwOI8MaEdjNciAQDk,3746
-pam/templates/service/polars/service_class.tmpl,sha256=8FDTFb6I5OHFqM0KPJtYDFAHeL3kl2WbwtbHAjwTG4Y,1830
-pam_python-0.2.2.dist-info/licenses/LICENSE.txt,sha256=yeHD2-gzrkkET2ZkMe22WyUGpDjlwTrqA4POb0FoZi4,1138
-pam_python-0.2.2.dist-info/METADATA,sha256=MWstqmu0l3XRLjBamIEM0p-uSf0UBHmX1-V2kczonVM,8669
-pam_python-0.2.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-pam_python-0.2.2.dist-info/entry_points.txt,sha256=xqYpsIqvNlO0j8elZc02bDwR-12dLK9FU2B1ACsLYxw,37
-pam_python-0.2.2.dist-info/top_level.txt,sha256=0EOjbyc3hQyzjhn6iyMgsEseqA66Xz0p27iBN7G7W1w,4
-pam_python-0.2.2.dist-info/RECORD,,
+pam/templates/service/polars/service.test.tmpl,sha256=2jduY9a2yILkSzKg9ViPo2_VQsVziXS1KdqFp_U6PUU,3937
+pam/templates/service/polars/service_class.tmpl,sha256=Pzw2Wcv1FZMDbWAcfakM2V0QhxKjq31mX5bfv5ekeTI,2666
+pam_python-0.2.4.dist-info/licenses/LICENSE.txt,sha256=yeHD2-gzrkkET2ZkMe22WyUGpDjlwTrqA4POb0FoZi4,1138
+pam_python-0.2.4.dist-info/METADATA,sha256=7i-8B4rbtCG13UGNa8Y3SNc-v7HwWQManVP9dw5kYjw,9025
+pam_python-0.2.4.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+pam_python-0.2.4.dist-info/entry_points.txt,sha256=xqYpsIqvNlO0j8elZc02bDwR-12dLK9FU2B1ACsLYxw,37
+pam_python-0.2.4.dist-info/top_level.txt,sha256=0EOjbyc3hQyzjhn6iyMgsEseqA66Xz0p27iBN7G7W1w,4
+pam_python-0.2.4.dist-info/RECORD,,