pam-python 0.1.29__tar.gz → 0.1.31__tar.gz

{pam_python-0.1.29 → pam_python-0.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.1.29
+Version: 0.1.31
 Summary: Pam Python Library
 Home-page: https://github.com/heart/pam-python
 Author: Narongrit Kanhanoi

{pam_python-0.1.29 → pam_python-0.1.31}/pam/cli.py

@@ -74,6 +74,7 @@ def create_service(name):
     cpy("service/service.yaml", os.path.join(name, "service.yaml"))
     cpy("service/functions.tmpl", os.path.join(name, "functions.py"))
     cpy("service/service.test.tmpl", os.path.join(name, f"test_{name}.py"))
+    
 
     replace_template_content(name, to_pascal_case(
         name)+"Svc", to_pascal_case(name)+"Svc.py")
@@ -96,6 +97,7 @@ def init_project():
     cpy("init/run_unit_test.sh", "run_unit_test.sh")
     cpy("init/run_unit_test.bat", "run_unit_test.bat")
     cpy("init/run_unit_test.ps1", "run_unit_test.ps1")
+    cpy("init/AGENT.md", "AGENT.md")
 
     if not os.path.exists("requirements.txt"):
         open("requirements.txt", 'a', encoding='utf-8').close()

{pam_python-0.1.29 → pam_python-0.1.31}/pam/interface_task_manager.py

@@ -18,7 +18,7 @@ class ITaskManager(ABC):
         pass
 
     @abstractmethod
-    def service_request_data(self, service: Service, page):
+    def service_request_data(self, service: Service, page, filter_contact_ids=None):
         pass
 
     @abstractmethod
@@ -30,7 +30,7 @@ class ITaskManager(ABC):
         pass
 
     @abstractmethod
-    def service_upload_result(self, service: Service, file_path):
+    def service_upload_result(self, service: Service, file_path, options=None):
         pass
 
     @abstractmethod

{pam_python-0.1.29 → pam_python-0.1.31}/pam/service.py

@@ -2,7 +2,7 @@ from abc import abstractmethod
 from pathlib import Path
 from datetime import datetime
 import json
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Union, Optional, List
 import pandas as pd
 import dask.dataframe as dd
 from pam.utils import log, deep_convert_numbers_to_strings, get_adapter_id
@@ -29,13 +29,17 @@ class Service:
         return get_adapter_id(self.request.response_api)
     
     # == REQUEST DATA ===
-    def _request_data(self, page: str = "") -> None:
+    def _request_data(self, page: str = "", filter_contact_ids: Optional[List[str]] = None) -> None:
         """Requests data for the specified page."""
         log(f"{self.request.service_name}: Requesting data for page={page}")
-        self.task_manager.service_request_data(self, page)
+        self.task_manager.service_request_data(self, page, filter_contact_ids)
 
     # == UPLOAD RESULT ===
-    def _upload_result(self, df: Union[pd.DataFrame, dd.DataFrame]) -> str:
+    def _upload_result(
+        self,
+        df: Union[pd.DataFrame, dd.DataFrame],
+        options: Optional[dict] = None,
+    ) -> str:
         """
         Uploads the result DataFrame to a temporary file and notifies the task manager.
         """
@@ -56,7 +60,7 @@ class Service:
         try:
             df.to_csv(tmp_file_name, index=False)
             log(f"{self.request.service_name}: Uploading result file: {tmp_file_name}")
-            self.task_manager.service_upload_result(self, tmp_file_name)
+            self.task_manager.service_upload_result(self, tmp_file_name, options)
         except Exception as e:
             log(f"Failed to upload result file: {e}")
             raise

{pam_python-0.1.29 → pam_python-0.1.31}/pam/task_manager.py

@@ -127,7 +127,7 @@ class TaskManager(ITaskManager):
             self._remove_service(token)
 
     # ==== Service Callbacks ====
-    def service_request_data(self, service: Service, page):
+    def service_request_data(self, service: Service, page, filter_contact_ids=None):
         """
         Makes an asynchronous request for data from a service.
         Logs the response without blocking the main thread.
@@ -135,6 +135,8 @@ class TaskManager(ITaskManager):
         endpoint = service.request.data_api
         token = service.request.token
         json_data = {"page": page, "token": token}
+        if filter_contact_ids is not None:
+            json_data["filter_contact_ids"] = filter_contact_ids
 
         def handle_response(response):
             """Logs the response from the API."""
@@ -225,11 +227,20 @@ class TaskManager(ITaskManager):
             log(f"{service.request.service_name}: Exception while uploading sqlite: {e}")
             return ""
 
-    def service_upload_result(self, service: Service, file_path):
+    def service_upload_result(self, service: Service, file_path, options=None):
         """
         Uploads a result file asynchronously and logs the response.
         """
         endpoint = service.request.response_api
+        payload = None
+        if options is not None:
+            payload = {}
+            if "is_realtime" in options:
+                is_realtime = options.get("is_realtime")
+                payload["is_realtime"] = str(is_realtime).lower() if isinstance(is_realtime, bool) else is_realtime
+            if "is_priority" in options:
+                is_priority = options.get("is_priority")
+                payload["is_priority"] = str(is_priority).lower() if isinstance(is_priority, bool) else is_priority
 
         def handle_upload_response(response):
             """Logs the response after the upload completes."""
@@ -241,7 +252,7 @@ class TaskManager(ITaskManager):
 
         def upload_wrapper():
             """Wrapper for the upload to handle response logging."""
-            response = self.api.http_upload(endpoint, file_path)
+            response = self.api.http_upload(endpoint, file_path, payload)
             handle_upload_response(response)
 
         log(f"Uploading Result to: {endpoint}")

pam_python-0.1.31/pam/templates/init/AGENT.md

@@ -0,0 +1,191 @@
+# AGENT
+
+This document explains the data plugin service lifecycle so developers can create new services and understand runtime behavior inside the CDP environment.
+
+## Project Overview
+
+1. This project is a data plugin that will be connected to the main system in production.
+2. The data plugin processes data from CDP and is created as separate services by domain. For example, if we process RFM, we create a new service dedicated to that domain.
+
+## Create a New Service
+
+1. Create a service with:
+   `pam new service [service name]`
+2. The system creates a new folder named `[service name]` with sample code and basic templates.
+3. Example:
+   `pam new service rfm_segment`
+   This generates `rfm_segment/Rfm_segmentSvc.py`, which is the starting point of the service.
+4. `rfm_segment/service.yaml` registers the service automatically.
+   The value `endpoint: /Rfm_segmentSvc` means CDP will call:
+   `http://ip-of-this-k8s-pod/services/Rfm_segmentSvc`
+
+## Service Lifecycle
+
+CDP spawns a service instance when a job starts and terminates it after the job completes. This design supports parallel execution. For example, two runs will spawn two separate instances. The only risk is shared local files (e.g., sqlite) which can cause conflicts.
+
+The lifecycle is wired in `rfm_segment/Rfm_segmentSvc.py` (example service name).
+
+### on_start
+
+`on_start` runs at startup. Use it to read runtime parameters from CDP and initialize work.
+
+Example:
+
+```python
+def on_start(self):
+    log("on_start")
+
+    some_param = self.request.runtime_parameters.get("some_param", "")
+    self._request_data()
+```
+
+`runtime_parameters` are configured by CDP to control service behavior without code changes (e.g., `period_day="7"`).
+
+Notes
+
+- All values are strings. Cast with defaults to avoid errors.
+- `dry_run` is the only built-in parameter. If missing, it runs for real. If `dry_run="true"`, the service runs fully but does not upload results to CDP.
+- You do not need to check `dry_run` manually. Always call `_upload_result`; the system handles it.
+
+**Important:** `on_start` must return quickly because CDP calls via HTTP. If it hangs, the client may time out and the job will be terminated. If work is long-running, move it to a thread and return from `on_start` first. This applies to all lifecycle functions.
+
+### Request Data from CDP
+
+Call `self._request_data()` to ask CDP for input data. CDP returns up to 5 CSV files (each file is a separate event).
+
+- File order is guaranteed per agreement, so you can index by position.
+- Column names and record counts are defined by CDP configuration.
+
+CSV example:
+
+```csv
+_id,id,data1,data2,data3
+xx,yy,zz,nn,mm
+```
+
+Column meaning
+
+- `_id` = event id for dedup/processing checks. Often unused and can be dropped.
+- `id` = customer id used by CDP as a key.
+- Other columns depend on the agreement.
+
+Date format is always:
+`"02/08/2026, 18:23:54"`
+Timezone may vary by customer data (usually Bangkok time).
+You may sometimes be asked for the exact time zone if it is logically important for the program.
+
+### on_data_input
+
+When CDP finishes collecting data, it calls `on_data_input`.
+
+Example:
+
+```python
+def on_data_input(self, req: RequestCommand):
+    log(f"on_data_input req.is_end = {req.is_end}")
+
+    # RUN IN A SEPARATED THREAD TO PREVENT HTTP REQUEST BLOCKING
+    thread = threading.Thread(
+        target=lambda: self.__run_process_data_in_thread(req),
+        daemon=True,
+    )
+    thread.start()
+
+
+def __run_process_data_in_thread(self, req: RequestCommand):
+    dataframe = self.__process_data(req.input_files)
+    self._upload_result(dataframe)
+
+    if not req.is_end:
+        self._request_data(req.next)
+    else:
+        self._exit()
+```
+
+Why a thread: to end the HTTP request quickly and avoid timeouts.
+
+`req.input_files` is the ordered list of CSV paths.
+
+### Upload Results to CDP
+
+After processing, call `_upload_result` with a dataframe. Example output:
+
+```csv
+id,data_x,rfm
+xxx,20,hero
+```
+
+- `id` is required for CDP matching.
+- Other columns are defined by agreement.
+
+Sometimes you may include an `event` column to create per-user events:
+
+```csv
+id,data_x,event
+xxx,20,purchase
+yyy,50,refund
+```
+
+If `event` is missing, CDP uses the default event configured in the system.
+
+### Pagination
+
+In the example:
+
+```python
+if not req.is_end:
+    self._request_data(req.next)
+else:
+    self._exit()
+```
+
+- `req.is_end` indicates whether this is the last page.
+- If not end, call `self._request_data(req.next)` for the next page.
+- If end, call `self._exit()` to terminate the instance.
+
+CDP will not send the next page until you call `_request_data`, so the flow is serialized and no two threads will handle the same data unless you explicitly design it that way.
+
+### on_terminate
+
+Before termination, the system calls:
+
+```python
+def on_terminate(self):
+    log("on_terminate")
+```
+
+Use it for cleanup.
+
+## Persist State Between Runs (sqlite/duckdb)
+
+Services can be killed and restarted at any time. If you need state across runs, use sqlite or duckdb via the provided API.
+
+Example (sqlite):
+
+```python
+SQLITE_FILE_NAME = "mydata.sqlite"
+
+sqlite_file = self._request_sqlite(file_name=SQLITE_FILE_NAME, is_shared=False)
+if sqlite_file is not None:
+    pass
+
+self._upload_sqlite(
+    file_name=SQLITE_FILE_NAME,
+    is_shared=False,
+    sqlite_file="/local/path/to/upload/sqlite.sqlite",
+)
+```
+
+Notes
+
+- `_request_sqlite` downloads from the server and is automatically scoped to this plugin. It will not mix with other plugins even if the name matches.
+- The call is synchronous and returns the local path.
+- If the return value is `None`, no file exists on the server; you can create a new one.
+- `_upload_sqlite` always overwrites the previous file.
+
+`is_shared` behavior
+
+- `False` (recommended): isolate per service. Multiple instances get their own copies.
+- `True`: shared storage. If another plugin updates the file, this plugin can download the latest version.
+
+Note: `file_name` acts like a server key, while `sqlite_file` is the local file path.

{pam_python-0.1.29 → pam_python-0.1.31}/pam/templates/service/service.test.tmpl

@@ -35,7 +35,7 @@ def create_mock_csv(page, mockDir):
         df.to_csv(file_name, index=False)
 
 
-def on_request_data(page) -> Tuple[List[str], bool, str]:
+def on_request_data(page, filter_contact_ids=None) -> Tuple[List[str], bool, str]:
     if page is None or page == "" :
         page = "1"
 

{pam_python-0.1.29 → pam_python-0.1.31}/pam/tester_task.py

@@ -4,7 +4,7 @@ from pam.interface_task_manager import ITaskManager
 from pam.service import Service
 
 
-RequestDataCallbackType = Callable[[str], Tuple[List[str], bool, str]]
+RequestDataCallbackType = Callable[..., Tuple[List[str], bool, str]]
 RequestSqliteCallbackType = Callable[[str, bool], Tuple[str]]
 UploadSqliteCallbackType = Callable[[str, bool, str], Tuple[str]]
 UploadResultCallbackType = Callable[[str], None]
@@ -81,16 +81,23 @@ class TesterTask(ITaskManager):
         while not self.is_exit:
             pass
 
-    def service_request_data(self, service: Service, page: str):
+    def service_request_data(self, service: Service, page: str, filter_contact_ids: Optional[List[str]] = None):
         """
         Simulate a service data request.
 
         :param service: The service requesting data.
         :param page: The page identifier for the data request.
+        :param filter_contact_ids: Optional list of contact IDs to filter the request.
         """
         if self.request_data_callback is not None:
             try:
-                files, is_end, next_page = self.request_data_callback(page)
+                if filter_contact_ids is None:
+                    files, is_end, next_page = self.request_data_callback(page)
+                else:
+                    try:
+                        files, is_end, next_page = self.request_data_callback(page, filter_contact_ids)
+                    except TypeError:
+                        files, is_end, next_page = self.request_data_callback(page)
                 req = RequestCommand(
                     sqlite_download="",
                     sqlite_upload="",
@@ -128,12 +135,18 @@ class TesterTask(ITaskManager):
         else:
             print("Upload sqlite callback is not set.")
 
-    def service_upload_result(self, service: Service, file_path: str):
+    def service_upload_result(
+        self,
+        service: Service,
+        file_path: str,
+        options: Optional[dict] = None,
+    ):
         """
         Simulate a service result upload.
 
         :param service: The service uploading the result.
         :param file_path: The path of the file being uploaded.
+        :param options: Optional upload options (e.g. is_realtime, is_priority).
         """
         if self.upload_result_callback is not None:
             try:

{pam_python-0.1.29 → pam_python-0.1.31}/pam_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pam-python
-Version: 0.1.29
+Version: 0.1.31
 Summary: Pam Python Library
 Home-page: https://github.com/heart/pam-python
 Author: Narongrit Kanhanoi

{pam_python-0.1.29 → pam_python-0.1.31}/pam_python.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ pam/models/request_command.py
 pam/templates/buildcmd/pamb
 pam/templates/buildcmd/pamb-base.sh
 pam/templates/docker/Dockerfile
+pam/templates/init/AGENT.md
 pam/templates/init/dockerignore.tmpl
 pam/templates/init/gitignore.tmpl
 pam/templates/init/main.tmpl

{pam_python-0.1.29 → pam_python-0.1.31}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="pam-python",
-    version="0.1.29",
+    version="0.1.31",
     author="Narongrit Kanhanoi",
     author_email="narongrit@pams.ai",
     description="Pam Python Library",