pam-python 0.2.5__tar.gz → 0.2.6__tar.gz

{pam_python-0.2.5/pam_python.egg-info → pam_python-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.2.5
+Version: 0.2.6
 Summary: Pam Python Library
 Author-email: Narongrit Kanhanoi <narongrit@pams.ai>
 Project-URL: Homepage, https://github.com/heart/pam-python
@@ -126,6 +126,17 @@ The runtime calls your service in two main phases.
 - `req.file_format` is the typed format sent by PAM
 - Should also return quickly (use a thread if needed)
 
+For customer-aligned Data Plugin Input V2, use `_request_data_v2(...)` and
+`req.data_input_v2`. The framework verifies the manifest and every physical
+part before plugin code sees them, and provides lazy Polars scans for CSV and
+Parquet (plus streaming row iterators for CSV).
+See [Data Plugin Input V2](docs/data-plugin-input-v2.md).
+
+New Pandas and Polars service projects are generated on the V2 lane by default;
+legacy source can continue using V1 unchanged. For materialized Pandas frames,
+use `data.read_population_pandas()` and
+`data.dataset_at(position).read_pandas()`.
+
 When your service is done:
 
 - Upload ordinary CDP result rows through `ResultBatchUploader`
@@ -135,12 +146,15 @@ When your service is done:
 ---
 
 **Using Temp Files Correctly**
-Temp storage is managed by the framework. Do not delete temp files manually.
+Temp storage is managed by the framework. Do not delete individual files
+manually. V2 code may release one completed page with
+`req.data_input_v2.cleanup()`; otherwise scheduled cleanup retains ownership.
 
 Standard helpers:
 
 - `TempfileUtils.get_temp_path_for_service(self, self.service_name)`
 - `TempfileUtils.get_temp_file_name_for_service(self, self.service_name, prefix, extension)`
+- `TempfileUtils.remove_dataset_bundle(path)` for one framework-owned page
 
 Notes:
 
@@ -269,6 +283,9 @@ Environment variables you can set:
 - `TEMP_DATASOURCE_PATH` (default `/app/data/data_sources`)
 - `TEMP_CLEAN_DAYS` (default `10`)
 - `TEMP_CLEAN_INTERVAL_HOURS` (default `6`, set empty to disable periodic cleanup)
+- `PAM_DATA_INPUT_V2_MAX_ZIP_BYTES` (default `1 GiB`)
+- `PAM_DATA_INPUT_V2_MAX_UNCOMPRESSED_BYTES` (default `2 GiB`)
+- `PAM_DATA_INPUT_V2_MAX_FILES` (default `10000`)
 
 ---
 

{pam_python-0.2.5 → pam_python-0.2.6}/README.md

@@ -106,6 +106,17 @@ The runtime calls your service in two main phases.
 - `req.file_format` is the typed format sent by PAM
 - Should also return quickly (use a thread if needed)
 
+For customer-aligned Data Plugin Input V2, use `_request_data_v2(...)` and
+`req.data_input_v2`. The framework verifies the manifest and every physical
+part before plugin code sees them, and provides lazy Polars scans for CSV and
+Parquet (plus streaming row iterators for CSV).
+See [Data Plugin Input V2](docs/data-plugin-input-v2.md).
+
+New Pandas and Polars service projects are generated on the V2 lane by default;
+legacy source can continue using V1 unchanged. For materialized Pandas frames,
+use `data.read_population_pandas()` and
+`data.dataset_at(position).read_pandas()`.
+
 When your service is done:
 
 - Upload ordinary CDP result rows through `ResultBatchUploader`
@@ -115,12 +126,15 @@ When your service is done:
 ---
 
 **Using Temp Files Correctly**
-Temp storage is managed by the framework. Do not delete temp files manually.
+Temp storage is managed by the framework. Do not delete individual files
+manually. V2 code may release one completed page with
+`req.data_input_v2.cleanup()`; otherwise scheduled cleanup retains ownership.
 
 Standard helpers:
 
 - `TempfileUtils.get_temp_path_for_service(self, self.service_name)`
 - `TempfileUtils.get_temp_file_name_for_service(self, self.service_name, prefix, extension)`
+- `TempfileUtils.remove_dataset_bundle(path)` for one framework-owned page
 
 Notes:
 
@@ -249,6 +263,9 @@ Environment variables you can set:
 - `TEMP_DATASOURCE_PATH` (default `/app/data/data_sources`)
 - `TEMP_CLEAN_DAYS` (default `10`)
 - `TEMP_CLEAN_INTERVAL_HOURS` (default `6`, set empty to disable periodic cleanup)
+- `PAM_DATA_INPUT_V2_MAX_ZIP_BYTES` (default `1 GiB`)
+- `PAM_DATA_INPUT_V2_MAX_UNCOMPRESSED_BYTES` (default `2 GiB`)
+- `PAM_DATA_INPUT_V2_MAX_FILES` (default `10000`)
 
 ---
 

{pam_python-0.2.5 → pam_python-0.2.6}/pam/__init__.py

@@ -1,3 +1,3 @@
 """PAM Data Plugin framework."""
 
-__version__ = "0.2.5"
+__version__ = "0.2.6"

{pam_python-0.2.5 → pam_python-0.2.6}/pam/api.py

@@ -26,7 +26,12 @@ class API:
             response.raise_for_status()
             return response
         except requests.RequestException as e:
-            log(f"HTTP POST request failed. URL: {url}, Error: {e}")
+            response_body = self.__error_response_body(e)
+            response_detail = f", Response: {response_body}" if response_body else ""
+            log(
+                f"HTTP POST request failed. URL: {url}, Error: {e}"
+                f"{response_detail}"
+            )
             return None
 
     def http_upload(self, url: str, file_path: str, payload: dict = None) -> requests.Response | None:
@@ -186,3 +191,15 @@ class API:
     def close(self):
         """Close the session."""
         self.session.close()
+
+    @staticmethod
+    def __error_response_body(error: requests.RequestException, limit: int = 2000) -> str:
+        """Return a bounded server error body for diagnostics."""
+        response = error.response
+        if response is None:
+            return ""
+
+        body = response.text.strip()
+        if len(body) <= limit:
+            return body
+        return f"{body[:limit]}... (truncated)"

{pam_python-0.2.5 → pam_python-0.2.6}/pam/interface_task_manager.py

@@ -15,6 +15,10 @@ class ITaskManager(ABC):
     def on_dataset_input(self, req: RequestCommand):
         pass
 
+    @abstractmethod
+    def on_dataset_error(self, req: RequestCommand) -> bool:
+        pass
+
     @abstractmethod
     def start_service(self, service_class, req: RequestCommand, service_name):
         pass
@@ -33,6 +37,16 @@ class ITaskManager(ABC):
     ):
         pass
 
+    @abstractmethod
+    def service_request_data_v2(
+        self,
+        service: Service,
+        page,
+        filter_contact_ids=None,
+        file_format: RequestFileFormat = RequestFileFormat.CSV,
+    ):
+        pass
+
     @abstractmethod
     def service_request_sqlite(self, service: Service, file_name: str = "", is_shared: bool = False):
         pass

pam_python-0.2.6/pam/models/data_input_v2.py

@@ -0,0 +1,424 @@
+from __future__ import annotations
+
+import csv
+from dataclasses import dataclass
+from datetime import datetime
+from hashlib import sha256
+import json
+from pathlib import Path
+from typing import Any, Iterator
+
+from pam.temp_file_utils import TempfileUtils
+
+
+V2_PROTOCOL_VERSION = "data-plugin-input/v2"
+V2_MANIFEST_FILE_NAME = "manifest.json"
+V2_MAX_DATASETS = 5
+V2_POPULATION_MODES = {"contacts", "event_any", "event_all"}
+
+
+@dataclass(frozen=True)
+class DataInputV2File:
+    path: str
+    format: str
+    row_count: int
+    size_bytes: int
+    sha256: str
+
+    def iter_csv_rows(self) -> Iterator[dict[str, str]]:
+        """Yield rows without loading this complete physical part into memory."""
+        if self.format != "csv":
+            raise ValueError(f"V2 file format is not CSV: {self.format}")
+        with open(self.path, "r", encoding="utf-8-sig", newline="") as input_file:
+            yield from csv.DictReader(input_file)
+
+    def scan_polars(self) -> Any:
+        """Return a lazy Polars scan selected from verified manifest metadata."""
+        pl = _import_polars()
+        if self.format == "parquet":
+            return pl.scan_parquet(self.path)
+        if self.format == "csv":
+            return pl.scan_csv(self.path)
+        raise ValueError(f"V2 file format is unsupported: {self.format}")
+
+    def read_pandas(self) -> Any:
+        """Materialize this verified file with the reader declared by metadata."""
+        pd = _import_pandas()
+        if self.format == "parquet":
+            return pd.read_parquet(self.path)
+        if self.format == "csv":
+            return pd.read_csv(self.path)
+        raise ValueError(f"V2 file format is unsupported: {self.format}")
+
+
+@dataclass(frozen=True)
+class DataInputV2Dataset:
+    position: int
+    name: str
+    type: str
+    format: str
+    row_count: int
+    event_names: tuple[str, ...]
+    parts: tuple[DataInputV2File, ...]
+
+    def iter_rows(self) -> Iterator[dict[str, str]]:
+        """Stream this logical dataset across its ordered physical CSV parts."""
+        if self.format.lower() != "csv":
+            raise ValueError(
+                "V2 dataset format is not supported for row iteration: "
+                f"{self.format}"
+            )
+        for part in self.parts:
+            yield from part.iter_csv_rows()
+
+    def scan_polars(self) -> Any:
+        """Return one lazy frame across all ordered CSV or Parquet parts."""
+        pl = _import_polars()
+        if not self.parts:
+            return pl.LazyFrame(schema={"id": pl.String})
+        scans = [part.scan_polars() for part in self.parts]
+        if len(scans) == 1:
+            return scans[0]
+        return pl.concat(scans, how="diagonal_relaxed")
+
+    def read_pandas(self) -> Any:
+        """Materialize and concatenate all verified parts using their metadata."""
+        pd = _import_pandas()
+        if not self.parts:
+            return pd.DataFrame({"id": pd.Series(dtype="string")})
+        frames = [part.read_pandas() for part in self.parts]
+        if len(frames) == 1:
+            return frames[0]
+        return pd.concat(frames, ignore_index=True, sort=False)
+
+
+@dataclass(frozen=True)
+class DataInputV2TimeWindow:
+    start: str
+    end: str
+    start_inclusive: bool
+    end_inclusive: bool
+
+
+@dataclass(frozen=True)
+class DataInputV2:
+    manifest_path: str
+    run_id: str
+    batch_id: str
+    batch_number: int
+    consistency: str
+    population_mode: str
+    max_contacts_per_batch: int
+    time_window: DataInputV2TimeWindow | None
+    population: DataInputV2File
+    datasets: tuple[DataInputV2Dataset, ...]
+    run_complete: bool
+    next: str
+
+    def iter_population_rows(self) -> Iterator[dict[str, str]]:
+        """Stream the population file for this customer-aligned logical page."""
+        yield from self.population.iter_csv_rows()
+
+    def scan_population_polars(self) -> Any:
+        """Return a lazy Polars scan for the authoritative population file."""
+        return self.population.scan_polars()
+
+    def read_population_pandas(self) -> Any:
+        """Materialize the authoritative population using manifest metadata."""
+        return self.population.read_pandas()
+
+    def cleanup(self) -> bool:
+        """Delete this page's extracted files and source ZIP.
+
+        Polars lazy frames must be collected before calling this method. When
+        cleanup is not called, the normal framework schedule retains ownership
+        of removing the files later.
+        """
+        return TempfileUtils.remove_dataset_bundle(self.manifest_path)
+
+    def dataset_at(self, position: int) -> DataInputV2Dataset:
+        """Bind one configured dataset by its zero-based contract position."""
+        if position < 0 or position >= len(self.datasets):
+            raise IndexError(f"V2 dataset position is out of range: {position}")
+        return self.datasets[position]
+
+
+def parse_data_input_v2(input_files: list[str]) -> DataInputV2:
+    files_by_name = _validate_flat_input_files(input_files)
+    manifest_path = files_by_name.get(V2_MANIFEST_FILE_NAME)
+    if manifest_path is None:
+        raise ValueError("V2 input is missing manifest.json")
+
+    try:
+        manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise ValueError("V2 manifest is invalid") from error
+    if not isinstance(manifest, dict):
+        raise ValueError("V2 manifest must be an object")
+    if manifest.get("protocol_version") != V2_PROTOCOL_VERSION:
+        raise ValueError("V2 manifest protocol_version is unsupported")
+    if manifest.get("flow_version") != "v2":
+        raise ValueError("V2 manifest flow_version is invalid")
+    if manifest.get("batch_complete") is not True:
+        raise ValueError("V2 manifest batch is incomplete")
+    consistency = _required_string(manifest, "consistency")
+    if consistency != "eventual":
+        raise ValueError("V2 manifest consistency is unsupported")
+    time_window = _parse_time_window(manifest.get("time_window"))
+
+    declared_names: set[str] = set()
+    population_block = _required_dict(manifest, "population")
+    population_mode = _required_string(population_block, "mode")
+    if population_mode not in V2_POPULATION_MODES:
+        raise ValueError("V2 manifest population mode is invalid")
+    max_contacts_per_batch = _required_positive_int(
+        population_block,
+        "max_contacts_per_batch",
+    )
+    population_format = _required_format(population_block, "format")
+    population_file = _parse_and_verify_file(
+        _required_dict(population_block, "file"),
+        files_by_name,
+        declared_names,
+        population_format,
+    )
+    contact_count = _required_non_negative_int(population_block, "contact_count")
+    if population_file.row_count != contact_count:
+        raise ValueError("V2 population row_count does not match contact_count")
+    if contact_count > max_contacts_per_batch:
+        raise ValueError("V2 population contact_count exceeds configured batch maximum")
+
+    raw_datasets = manifest.get("datasets")
+    if not isinstance(raw_datasets, list) or len(raw_datasets) > V2_MAX_DATASETS:
+        raise ValueError("V2 manifest datasets must be a list of at most five items")
+    datasets: list[DataInputV2Dataset] = []
+    positions: set[int] = set()
+    for raw_dataset in raw_datasets:
+        if not isinstance(raw_dataset, dict):
+            raise ValueError("V2 dataset entry must be an object")
+        if raw_dataset.get("complete") is not True:
+            raise ValueError("V2 dataset is incomplete")
+        position = _required_non_negative_int(raw_dataset, "position")
+        if position in positions:
+            raise ValueError("V2 dataset position is duplicated")
+        positions.add(position)
+        dataset_format = _required_format(raw_dataset, "format")
+        raw_parts = raw_dataset.get("parts")
+        if not isinstance(raw_parts, list):
+            raise ValueError("V2 dataset parts must be a list")
+        parts = tuple(
+            _parse_and_verify_file(
+                part,
+                files_by_name,
+                declared_names,
+                dataset_format,
+            )
+            for part in raw_parts
+        )
+        row_count = _required_non_negative_int(raw_dataset, "row_count")
+        if sum(part.row_count for part in parts) != row_count:
+            raise ValueError("V2 dataset part row counts do not match dataset row_count")
+        event_names = raw_dataset.get("event_names", [])
+        if not isinstance(event_names, list) or not all(
+            isinstance(event_name, str) for event_name in event_names
+        ):
+            raise ValueError("V2 dataset event_names must be a list of strings")
+        datasets.append(
+            DataInputV2Dataset(
+                position=position,
+                name=_required_string(raw_dataset, "name"),
+                type=_required_string(raw_dataset, "type"),
+                format=dataset_format,
+                row_count=row_count,
+                event_names=tuple(event_names),
+                parts=parts,
+            )
+        )
+
+    datasets.sort(key=lambda dataset: dataset.position)
+    if [dataset.position for dataset in datasets] != list(range(len(datasets))):
+        raise ValueError("V2 dataset positions must be contiguous and zero-based")
+    has_event_dataset = any(dataset.type == "event" for dataset in datasets)
+    if has_event_dataset and time_window is None:
+        raise ValueError("V2 event datasets require a time_window")
+    if population_mode != "contacts" and not has_event_dataset:
+        raise ValueError("V2 event population mode requires an event dataset")
+
+    actual_data_names = set(files_by_name) - {V2_MANIFEST_FILE_NAME}
+    if declared_names != actual_data_names:
+        missing = declared_names - actual_data_names
+        extra = actual_data_names - declared_names
+        raise ValueError(f"V2 manifest file set mismatch: missing={missing}, extra={extra}")
+
+    return DataInputV2(
+        manifest_path=manifest_path,
+        run_id=_required_string(manifest, "run_id"),
+        batch_id=_required_string(manifest, "batch_id"),
+        batch_number=_required_positive_int(manifest, "batch_number"),
+        consistency=consistency,
+        population_mode=population_mode,
+        max_contacts_per_batch=max_contacts_per_batch,
+        time_window=time_window,
+        population=population_file,
+        datasets=tuple(datasets),
+        run_complete=_required_bool(manifest, "run_complete"),
+        next=_required_optional_string(manifest, "next"),
+    )
+
+
+def _validate_flat_input_files(input_files: list[str]) -> dict[str, str]:
+    files_by_name: dict[str, str] = {}
+    parent: Path | None = None
+    for raw_path in input_files:
+        path = Path(raw_path).resolve()
+        if not path.is_file():
+            raise ValueError("V2 input contains a missing file")
+        if parent is None:
+            parent = path.parent
+        elif path.parent != parent:
+            raise ValueError("V2 input files must share one extraction directory")
+        if path.name in files_by_name:
+            raise ValueError("V2 input contains a duplicate file name")
+        files_by_name[path.name] = str(path)
+    return files_by_name
+
+
+def _parse_and_verify_file(
+    raw_file: Any,
+    files_by_name: dict[str, str],
+    declared_names: set[str],
+    file_format: str,
+) -> DataInputV2File:
+    if not isinstance(raw_file, dict):
+        raise ValueError("V2 file metadata must be an object")
+    relative_path = _required_string(raw_file, "path")
+    if Path(relative_path).name != relative_path or "/" in relative_path or "\\" in relative_path:
+        raise ValueError("V2 file path must be a safe flat file name")
+    if relative_path == V2_MANIFEST_FILE_NAME or relative_path in declared_names:
+        raise ValueError("V2 file path is duplicated or reserved")
+    actual_path = files_by_name.get(relative_path)
+    if actual_path is None:
+        raise ValueError(f"V2 declared file is missing: {relative_path}")
+    if Path(relative_path).suffix.lower() != f".{file_format}":
+        raise ValueError(f"V2 file extension does not match format: {relative_path}")
+
+    expected_size = _required_non_negative_int(raw_file, "size_bytes")
+    actual_size = Path(actual_path).stat().st_size
+    if actual_size != expected_size:
+        raise ValueError(f"V2 file size mismatch: {relative_path}")
+    expected_hash = _required_string(raw_file, "sha256").lower()
+    actual_hash = _sha256_file(actual_path)
+    if actual_hash != expected_hash:
+        raise ValueError(f"V2 file SHA-256 mismatch: {relative_path}")
+
+    declared_names.add(relative_path)
+    return DataInputV2File(
+        path=actual_path,
+        format=file_format,
+        row_count=_required_non_negative_int(raw_file, "row_count"),
+        size_bytes=actual_size,
+        sha256=actual_hash,
+    )
+
+
+def _sha256_file(file_path: str) -> str:
+    digest = sha256()
+    with open(file_path, "rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _required_dict(source: dict[str, Any], key: str) -> dict[str, Any]:
+    value = source.get(key)
+    if not isinstance(value, dict):
+        raise ValueError(f"V2 manifest {key} must be an object")
+    return value
+
+
+def _required_string(source: dict[str, Any], key: str) -> str:
+    value = source.get(key)
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"V2 manifest {key} must be a non-empty string")
+    return value
+
+
+def _required_format(source: dict[str, Any], key: str) -> str:
+    value = _required_string(source, key).lower()
+    if value not in {"csv", "parquet"}:
+        raise ValueError(f"V2 manifest {key} must be csv or parquet")
+    return value
+
+
+def _required_bool(source: dict[str, Any], key: str) -> bool:
+    value = source.get(key)
+    if not isinstance(value, bool):
+        raise ValueError(f"V2 manifest {key} must be a boolean")
+    return value
+
+
+def _required_optional_string(source: dict[str, Any], key: str) -> str:
+    value = source.get(key)
+    if not isinstance(value, str):
+        raise ValueError(f"V2 manifest {key} must be a string")
+    return value
+
+
+def _parse_time_window(raw_time_window: Any) -> DataInputV2TimeWindow | None:
+    if raw_time_window is None:
+        return None
+    if not isinstance(raw_time_window, dict):
+        raise ValueError("V2 manifest time_window must be an object")
+    start = _required_string(raw_time_window, "start")
+    end = _required_string(raw_time_window, "end")
+    start_inclusive = _required_bool(raw_time_window, "start_inclusive")
+    end_inclusive = _required_bool(raw_time_window, "end_inclusive")
+    try:
+        start_time = datetime.fromisoformat(start.replace("Z", "+00:00"))
+        end_time = datetime.fromisoformat(end.replace("Z", "+00:00"))
+    except ValueError as error:
+        raise ValueError("V2 manifest time_window timestamps are invalid") from error
+    if start_time.tzinfo is None or end_time.tzinfo is None or start_time >= end_time:
+        raise ValueError("V2 manifest time_window order is invalid")
+    if not start_inclusive or end_inclusive:
+        raise ValueError("V2 manifest time_window must be half-open")
+    return DataInputV2TimeWindow(
+        start=start,
+        end=end,
+        start_inclusive=start_inclusive,
+        end_inclusive=end_inclusive,
+    )
+
+
+def _import_polars() -> Any:
+    try:
+        import polars as pl
+    except ImportError as error:
+        raise RuntimeError(
+            "Polars is required for V2 scan_polars(); add polars to the plugin dependencies"
+        ) from error
+    return pl
+
+
+def _import_pandas() -> Any:
+    try:
+        import pandas as pd
+    except ImportError as error:
+        raise RuntimeError(
+            "Pandas is required for V2 pandas readers; install the pandas dependency"
+        ) from error
+    return pd
+
+
+def _required_non_negative_int(source: dict[str, Any], key: str) -> int:
+    value = source.get(key)
+    if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+        raise ValueError(f"V2 manifest {key} must be a non-negative integer")
+    return value
+
+
+def _required_positive_int(source: dict[str, Any], key: str) -> int:
+    value = _required_non_negative_int(source, key)
+    if value == 0:
+        raise ValueError(f"V2 manifest {key} must be a positive integer")
+    return value