pam-python 0.1.0__py3-none-any.whl

pam/api.py

@@ -0,0 +1,42 @@
+import requests
+from pam.utils import log
+
+
+class API:
+
+    def http_post(self, url, data):
+        """
+        Sends an HTTP POST request to the specified URL with the given data as JSON.
+
+        :param url: The URL to send the POST request to.
+        :param data: A dictionary to be used as the JSON body of the POST request.
+        :return: The response from the server.
+        """
+        headers = {
+            'Content-Type': 'application/json'}
+        try:
+            response = requests.post(
+                url, json=data, timeout=30, headers=headers)
+            response.raise_for_status()  # Raise an exception for HTTP errors
+            return response
+        except requests.RequestException as e:
+            log(f"HTTP POST request failed: {e}")
+            return None
+
+    def http_upload(self, url, file_path):
+        """
+        Uploads a file to the specified URL.
+
+        :param url: The URL to upload the file to.
+        :param file_path: The path to the file to be uploaded.
+        :return: The response from the server.
+        """
+        try:
+            with open(file_path, 'rb') as file:
+                files = {'file': file}
+                response = requests.post(url, files=files, timeout=300)
+                response.raise_for_status()  # Raise an exception for HTTP errors
+                return response
+        except requests.RequestException as e:
+            log(f"File upload failed: {e}")
+            return None

pam/cli.py

@@ -0,0 +1,107 @@
+import os
+import sys
+import pkg_resources 
+import shutil
+import glob
+import subprocess
+
+def main():
+    args = sys.argv[1:]
+    if len(args) == 0:
+        print("Usage: pam <command> [args]")
+        return
+    
+    cmd = args[0]
+
+    if cmd == "init":
+        init_project()
+    elif cmd == "new":
+        type = args[1]
+        if type == "service":
+            name = args[2]
+            create_service(name)
+    elif cmd == "test":
+        if len(args) < 2:
+            print("Usage: pam test <modulename>")
+            return
+        module_name = args[1]
+        test_module(module_name)
+    else:
+        print(f"Unknown command: {args.command}")
+
+def test_module(module_name):
+    """Run all test files matching '*.test.py' in the specified module directory."""
+    # Locate all test files in the module directory
+    test_files = glob.glob(f"{module_name}/*.test.py")
+    
+    if not test_files:
+        print(f"Error: No test files found in module '{module_name}' (expected '*.test.py').")
+        return
+    
+    print(f"Found {len(test_files)} test file(s) in '{module_name}':")
+    for test_file in test_files:
+        print(f" - {test_file}")
+    
+    # Run each test file
+    for test_file in test_files:
+        print(f"\nRunning tests in {test_file}...")
+        result = subprocess.run([sys.executable, test_file], capture_output=True, text=True)
+
+        print("\nTest Output:")
+        print(result.stdout)
+        if result.stderr:
+            print("\nErrors:")
+            print(result.stderr)
+        print("-" * 40)  # Separator for clarity
+
+def cpy(src, dest):
+    template_dir = pkg_resources.resource_filename("pam", "templates")
+    src_file = os.path.join(template_dir, src)
+    shutil.copy(src_file, dest)
+
+def replace_template_content(service_name, file_name):
+    file_path = os.path.join(service_name, file_name)
+    with open(file_path, 'r+') as file:
+        filedata = file.read()
+        updated_data = filedata.replace('#CLASS_NAME#', service_name)
+        file.seek(0)  # Move the file pointer to the beginning of the file
+        file.write(updated_data)
+        file.truncate()  # Remove any leftover content after the replacement
+
+def create_service(name):
+    if os.path.exists(name):
+        response = input(f"Service {name} already exists. Do you want to overwrite it? (y/N): ").strip().lower()
+        if response == 'y':
+            shutil.rmtree(name)
+        else:
+            print("Cancelled.")
+            return
+    
+    os.mkdir(name)
+    open(os.path.join(name, "__init__.py"), 'a', encoding='utf-8').close()
+    cpy("service/ServiceClass.tmpl", os.path.join(name, name+"_service.py") )
+    cpy("service/service.yaml", os.path.join(name, "service.yaml") )
+    cpy("service/functions.tmpl", os.path.join(name, "functions.py") )
+    cpy("service/service.test.tmpl", os.path.join(name, name+".test.py") )
+
+    
+    replace_template_content(name, name+"_service.py")
+    replace_template_content(name, "service.yaml")
+    replace_template_content(name, name+".test.py")
+
+    print(f"Service {name} created.")
+    print(f"Run `pam test {name}` to run tests for the service.")
+
+def init_project():
+    cpy("init/main.tmpl", "main.py")
+    cpy("docker/Dockerfile", "Dockerfile")
+    cpy("buildcmd/pamb", "pamb")
+    cpy("buildcmd/pamb-base.sh", "pamb-base.sh")
+    if not os.path.exists("requirements.txt"):
+        open("requirements.txt", 'a', encoding='utf-8').close()
+    
+    with open("requirements.txt", "w", encoding='utf-8') as f:
+        subprocess.run(["pip", "freeze"], stdout=f, check=True)
+
+    print("--- Welcome to PAM ---\n")
+    print("To create a new servive run\n`pam new service <service_name>`\n\n")

pam/interface_task_manager.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from pam.models.request_command import RequestCommand
+from pam.service import Service
+
+
+class ITaskManager(ABC):
+
+    @abstractmethod
+    def on_dataset_input(self, req: RequestCommand):
+        pass
+
+    @abstractmethod
+    def start_service(self, service_class, req: RequestCommand, service_name):
+        pass
+
+    @abstractmethod
+    def terminate_service(self, token):
+        pass
+
+    @abstractmethod
+    def service_request_data(self, service: Service, page):
+        pass
+
+    @abstractmethod
+    def service_upload_result(self, service: Service, file_path):
+        pass
+
+    @abstractmethod
+    def service_upload_report(self, service: Service, file_path):
+        pass

pam/models/request_command.py

@@ -0,0 +1,138 @@
+
+"""
+RequestCommand is object that Wrap the http request to object that simple to access the parameters
+"""
+
+import os
+from dataclasses import dataclass
+import zipfile
+from pathlib import Path
+from pam.utils import log
+from pam.temp_file_utils import TempfileUtils
+
+
+@dataclass
+class RequestCommand:
+    """
+    RequestCommand is object that Wrap the http request
+    """
+
+    token: str
+    cmd: str
+    data_request_api: str
+    response_api: str
+    is_end: bool
+    next: str
+    input_files: list[str]
+    service_name: str
+
+    def is_start_command(self):
+        """Determine is this command is start command"""
+        return self.cmd == "start"
+
+    def is_dataset_command(self):
+        """Determine is this command is dataset command"""
+        return self.cmd == "dataset"
+
+    @staticmethod
+    def __safe_str_to_bool(value):
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            if value.lower() in ['true', '1', 'yes']:
+                return True
+            if value.lower() in ['false', '0', 'no']:
+                return False
+        return False
+
+    @staticmethod
+    def __extract_data_from_request(request, keys):
+        result = {}
+
+        if request.content_type == 'application/json':
+            data = request.get_json()
+            for key in keys:
+                if key in data:
+                    result[key] = data[key]
+                else:
+                    result[key] = ""
+        elif request.content_type.startswith('multipart/form-data'):
+            for key in keys:
+                data_str = request.form.get(key, '')
+                if data_str:
+                    result[key] = data_str
+                else:
+                    result[key] = ""
+
+        return result
+
+    @staticmethod
+    def __allowed_file(filename):
+        return '.' in filename and filename.rsplit(
+            '.', 1)[1].lower() in {'zip'}
+
+    @staticmethod
+    def __extract_input_file(zip_file, service_name, token):
+
+        if zip_file.filename == '':
+            log("handle_file_upload 'No selected file'")
+            return ([], "No selected file")
+
+        if not RequestCommand.__allowed_file(zip_file.filename):
+            log("handle_file_upload 'Invalid file type'")
+            return ([], "Invalid file type")
+
+        zip_file_dir = TempfileUtils.get_temp_file_name(
+            service_name, token, "dataset_")
+
+        zip_file_name = f"{zip_file_dir}.zip"
+        zip_file.save(zip_file_name)
+
+        extract_dir = Path(zip_file_dir)
+        extract_dir.mkdir(parents=True, exist_ok=True)
+
+        with zipfile.ZipFile(zip_file_name, 'r') as zip_ref:
+
+            zip_ref.extractall(zip_file_dir)
+
+            if os.path.exists(f"{extract_dir}/__MACOSX"):
+                os.rmdir(f"{extract_dir}/__MACOSX")
+            if os.path.exists(f"{extract_dir}/.DS_Store"):
+                os.rmdir(f"{extract_dir}/.DS_Store")
+
+        input_files = sorted([os.path.join(extract_dir, name)
+                              for name in os.listdir(extract_dir)])
+
+        log(f"Extract dir: {extract_dir}")
+        log(f"File save to: {input_files}")
+
+        return (input_files, "")
+
+    @staticmethod
+    def parse(request, service_name):
+        """Create RequestCommand object from http request"""
+        params = RequestCommand.__extract_data_from_request(
+            request, ["cmd", "token", "data", "response", "is_end", "next"])
+
+        token = params["token"]
+        cmd = params["cmd"]
+        data = params["data"]
+        response = params["response"]
+        is_end = RequestCommand.__safe_str_to_bool(params["is_end"])
+        next_page = params["next"]
+        input_files = []
+        error_message = ""
+
+        if not token:
+            error_message = 'The `token` parameter is required.'
+        else:
+            if cmd == "dataset":
+                zip_file = request.files['file']
+                (files, error) = RequestCommand.__extract_input_file(
+                    zip_file, service_name, token)
+                error_message = error
+                input_files = files
+
+        return RequestCommand(
+            token, cmd, data, response, is_end, next_page, input_files, service_name
+        ), error_message

pam/server.py

@@ -0,0 +1,170 @@
+"""
+Data Plugin Server
+"""
+
+import os
+import importlib
+import yaml
+from setuptools import find_packages
+from flask import request, make_response, Flask, jsonify
+from pam.utils import log
+from pam.models.request_command import RequestCommand
+from pam.task_manager import TaskManager
+from pam.temp_file_utils import TempfileUtils
+
+
+class Server:
+    """
+    Data Plugin Server
+    """
+
+    app: Flask
+    servicePool = {}
+
+    def __init__(self, app):
+        self.task_manager = TaskManager(self)
+        self.app = app
+        self.register_service()
+        self.task_manager.start_service_monitoring_schedul()
+
+        @app.route('/', methods=['GET'])
+        def home():
+            return self.response_ok({'message': "Hello, Data plugin v2."})
+
+        @app.route('/service/<service_name>', methods=['POST'])
+        def run(service_name):
+            TempfileUtils.clean_temp()
+
+            req, error_request = RequestCommand.parse(request, service_name)
+
+            if error_request != "":
+                response = {
+                    'message': error_request
+                }
+                return self.response_error(response, 400)
+
+            response, code = self.handle_service_cmd(service_name, req)
+            return self.response_error(response, code)
+
+        @app.route('/tasks', methods=['GET'])
+        def tasks():
+            return self.response_ok({'message': "OK"})
+
+        @app.route('/status/<service_name>', methods=['GET'])
+        def ping(service_name):
+            # TODO: check the service status
+            return self.response_ok({'message': f"OK {service_name}"})
+
+    def get_service_class(self, service_name):
+        """
+        Get service class from service name
+        """
+        if service_name in self.servicePool:
+            return self.servicePool[service_name]
+
+        return None
+
+    def handle_service_cmd(self, service_name, req: RequestCommand):
+        """
+        Handle all cmd type
+        """
+        service_class = self.get_service_class(service_name)
+
+        if service_class is not None:
+            if req.is_start_command():
+                log(f"Request on start {service_name} token={req.token}")
+                return self.on_start(service_class, req, service_name)
+
+            if req.is_dataset_command():
+
+                log(f"""Request on dataset {service_name} token={
+                    req.token} files={req.input_files}""")
+
+                return self.on_dataset(req)
+
+        response = {'message': f'Service `{service_name}` Notfound. '}
+        return response, 404
+
+    def on_start(self, service_class, req, service_name):
+        """
+        start a service
+        """
+        log(f"{service_name} -> on_start")
+        self.task_manager.start_service(service_class, req, service_name)
+        return {'acknowledge': True}, 200
+
+    def on_dataset(self, req):
+        """
+        task_manager handle cmd=dataset
+        """
+        log(f"{req.service_name} -> on_dataset")
+        self.task_manager.on_dataset_input(req)
+        return {'acknowledge': True}, 200
+
+    def on_register_service(self, service, end_point):
+        """
+        register a service
+        """
+        log(f"register service \"{service}\", endpoint: http://localhost/service/{end_point}")
+        self.servicePool[end_point] = service
+
+    def run(self, host='0.0.0.0', port=8000):
+        """
+        run flask app
+        """
+        self.app.run(host=host, port=port)
+
+    def response_error(self, json, code):
+        """
+        create error response with custom http status code
+        """
+        data = jsonify(json)
+        return make_response(data, code)
+
+    def response_ok(self, json, code=200):
+        """
+        create response object with 200OK
+        """
+        data = jsonify(json)
+        return make_response(data, code)
+
+    def get_service_config(self, package):
+        """
+        load the service config from service.yml
+        """
+        config_path_1 = f'{package}/service.yaml'
+        config_path_2 = f'{package}/service.yml'
+        if os.path.exists(config_path_1) and os.path.isfile(config_path_1):
+            with open(config_path_1, 'r', encoding='utf-8') as file:
+                config = yaml.safe_load(file)
+                return config
+        elif os.path.exists(config_path_2) and os.path.isfile(config_path_2):
+            with open(config_path_2, 'r', encoding='utf-8') as file:
+                config = yaml.safe_load(file)
+                return config
+        return None
+
+    def register_service(self):
+        """
+        scan project to register all plugin services
+        """
+        packages = find_packages()
+        for package in packages:
+            config = self.get_service_config(package)
+            if config is not None:
+                endpoint = config['endpoint']
+                class_name = config['class']
+                module = importlib.import_module(
+                    f'{package}.{class_name}')
+
+                try:
+                    class_ = getattr(module, class_name)
+                except AttributeError:
+                    log(
+                        f"Error: Class '{class_name}' not found in module '{module}'.")
+                    class_ = None
+
+                if endpoint.startswith("/"):
+                    endpoint = endpoint[1:]
+
+                self.on_register_service(class_, endpoint)

pam/service.py

@@ -0,0 +1,101 @@
+"""
+Service base class
+"""
+
+import os
+import tempfile
+import uuid
+from abc import abstractmethod
+import json
+from typing import TYPE_CHECKING
+import pandas as pd
+import dask.dataframe as dd
+from pam.utils import log, deep_convert_numbers_to_strings, get_adapter_id
+from pam.models.request_command import RequestCommand
+from pam.temp_file_utils import TempfileUtils
+
+if TYPE_CHECKING:
+    from pam.task_manager import TaskManager
+
+
+class Service:
+    """
+    Service base class
+    """
+
+    # pylint: disable=too-many-instance-attributes
+    def __init__(self, task_manager: 'TaskManager', req: RequestCommand):
+        self.request = req
+        self.task_manager = task_manager
+
+    # == REQUEST DATA ===
+    def _request_data(self, page=""):
+        log(f"{self.request.service_name}: Request data page={page}")
+        self.task_manager.service_request_data(self, page)
+
+    # == UPLOAD RESULT ===
+    def _upload_result(self, df):
+        if isinstance(df, dd.DataFrame):
+            df = df.compute()
+        elif not isinstance(df, pd.DataFrame):
+            raise ValueError("The input must be a Pandas or Dask DataFrame")
+
+        tmp_file_name = TempfileUtils.get_temp_file_name(
+            self.request.service_name, self.request.token, "result_", ".csv")
+
+        df.to_csv(tmp_file_name, index=False)
+
+        log(f"{self.request.service_name}: Upload Result File {tmp_file_name}")
+        self.task_manager.service_upload_result(self, tmp_file_name)
+
+        return tmp_file_name
+
+    # == UPLOAD REPORT ===
+
+    def _upload_report(self, report_name: str, report_json: str):
+        adapter_id = get_adapter_id(self.request.response_api)
+        service_name = self.request.service_name
+        token = self.request.token
+
+        # important need to recursive and convert numbers to str
+        report_json = deep_convert_numbers_to_strings(
+            report_json)
+
+        json_string = json.dumps(report_json)
+
+        report_name = f"data_{report_name}_no_index"
+
+        df = pd.DataFrame({
+            'customer': [adapter_id],
+            report_name: [json_string]
+        })
+
+        report_csv_path = TempfileUtils.get_temp_file_name(
+            service_name, token, f"report_{report_name}_", '.csv')
+
+        df.to_csv(report_csv_path, index=False)
+        self.task_manager.service_upload_report(self, report_csv_path)
+    # =========
+
+    def _exit(self):
+        self.task_manager.service_exit(self)
+
+    @abstractmethod
+    def on_start(self):
+        """on sstart"""
+
+    @abstractmethod
+    def on_data_input(self, req):
+        """on_data_input"""
+
+    @abstractmethod
+    def on_destroy(self):
+        """on_destroy"""
+
+    @abstractmethod
+    def on_terminate(self):
+        """on_terminate"""
+
+    @abstractmethod
+    def get_status(self):
+        """get_status"""

pam/task_manager.py

@@ -0,0 +1,141 @@
+from datetime import datetime, timedelta
+import time
+import threading
+from typing import Dict
+from pam.utils import log
+from pam.api import API
+from pam.service import Service
+from pam.models.request_command import RequestCommand
+from pam.interface_task_manager import ITaskManager
+from pam.temp_file_utils import TempfileUtils
+
+
+class ServiceHolder:
+    def __init__(self, service: Service):
+        self.service = service
+        self.last_activity = datetime.now()
+
+    def update_timestamp(self):
+        self.last_activity = datetime.now()
+
+    def has_timed_out(self, timeout=2):
+        # ตรวจสอบว่า service นี้ไม่เคลื่อนไหวเกินกว่า timeout ชั่วโมงหรือไม่
+        return datetime.now() - self.last_activity > timedelta(hours=timeout)
+
+
+class TaskManager(ITaskManager):
+    """
+    Manage service thread
+    """
+
+    def __init__(self, server):
+        self.services: Dict[str, ServiceHolder] = {}
+        self.api = API()
+        self.server = server
+        self.thread_lock = threading.Lock()
+
+    def _add_service(self, token, service):
+        with self.thread_lock:
+            self.services[token] = ServiceHolder(service)
+
+    def _get_service_holder(self, token):
+        with self.thread_lock:
+            return self.services.get(token)
+
+    def _update_service(self, token):
+        with self.thread_lock:
+            if token in self.services:
+                self.services[token].update_timestamp()
+
+    def _check_timeout(self, token, timeout=2):
+        with self.thread_lock:
+            if token in self.services:
+                return self.services[token].has_timed_out(timeout)
+        return False
+
+    def _remove_service(self, token):
+        with self.thread_lock:
+            if token in self.services:
+                service_holder = self.services[token]
+                log(f"Service Exit: {
+                    service_holder.service.request.service_name}, {service_holder.service.request.token}")
+                service_holder.service.on_destroy()
+                del self.services[token]
+
+    def start_service_monitoring_schedul(self):
+        schedule_thread = threading.Thread(target=self._monitor_services)
+        schedule_thread.daemon = True
+        schedule_thread.start()
+
+    def _monitor_services(self, interval=600, timeout=2):
+        while True:
+            time.sleep(interval)
+            log("Service Monitor running.")
+            tokens_to_remove = [
+                token for token in self.services if self._check_timeout(token, timeout)
+            ]
+            log(f"Found: {len(tokens_to_remove)} services timeout.")
+            for token in tokens_to_remove:
+                log(f"Service {token} has timed out. Removing...")
+                self._remove_service(token)
+
+    def on_dataset_input(self, req: RequestCommand):
+        """
+        handle cmd dataset
+        """
+        service_holder = self._get_service_holder(req.token)
+        if service_holder is not None:
+            service_holder.update_timestamp()
+            service_holder.service.on_data_input(req)
+
+    def start_service(self, service_class, req: RequestCommand, service_name):
+        """
+        Start new service
+        """
+        log(f"Start Services {service_name}, Token: {req.token}")
+        service_instance = service_class(self, req)
+        service_instance.on_start()
+        self._add_service(req.token, service_instance)
+
+    def terminate_service(self, token):
+        """
+        stop service that started from a token
+        """
+        service_holder = self._get_service_holder(token)
+        if service_holder is not None:
+            service_holder.service.on_terminate()
+            self._remove_service(token)
+
+    # ======= Service Callback function ======
+    def service_request_data(self, service: Service, page):
+        endpoint = service.request.data_request_api
+        token = service.request.token
+
+        json_data = {"page": page, "token": token}
+
+        log(f"Request Data to: {endpoint}, page: {page}, token={token}")
+        http_thread = threading.Thread(
+            target=self.api.http_post, args=(endpoint, json_data, ))
+        http_thread.start()
+        http_thread.join()
+
+    def service_upload_result(self, service: Service, file_path):
+        endpoint = service.request.response_api
+
+        log(f"Upload Data to: {endpoint}")
+        http_thread = threading.Thread(
+            target=self.api.http_upload, args=(endpoint, file_path, ))
+        http_thread.start()
+        http_thread.join()
+
+    def service_upload_report(self, service: Service, file_path):
+        endpoint = service.request.response_api
+
+        log(f"Upload Report to: {endpoint}")
+        http_thread = threading.Thread(
+            target=self.api.http_upload, args=(endpoint, file_path, ))
+        http_thread.start()
+        http_thread.join()
+
+    def service_exit(self, service: Service):
+        self._remove_service(service.request.token)

pam/temp_file_utils.py

@@ -0,0 +1,75 @@
+from pathlib import Path
+from datetime import datetime, timedelta
+import shutil
+import uuid
+from pam.utils import log
+
+
+def clean_old_folders(days: int):
+
+    base_folder = Path(TempfileUtils.temp_datasource_path)
+
+    if not base_folder.exists():
+        log("No temp dir to clean")
+        return
+    elif base_folder.is_file():
+        log("Something wrong, temp dir is not folder. Plugin will delete it.")
+        base_folder.unlink()
+        return
+
+    cutoff_date = datetime.now() - timedelta(days=days)
+
+    # Iterate through all subfolders in the base_folder
+    for folder in base_folder.iterdir():
+        if folder.is_dir():
+            try:
+                # Extract the date from the folder name
+                folder_date = datetime.strptime(folder.name, "%Y_%m_%d")
+
+                # If the folder is older than the cutoff date, remove it
+                if folder_date < cutoff_date:
+                    shutil.rmtree(folder)
+                    log(f"Removed folder: {folder}")
+            except ValueError:
+                # If folder name doesn't match the expected format, skip it
+                log(f"""Skipping folder: {
+                    folder} (unexpected name format)""")
+
+
+class TempfileUtils:
+
+    temp_base_path = "/app/data"
+    temp_datasource_path = "/app/data/data_sources"
+
+    @staticmethod
+    def clean_temp():
+        clean_old_folders(10)
+
+    @staticmethod
+    def get_temp_path(service_name, token):
+        date_path = datetime.now().strftime("%Y_%m_%d")
+        temp_dir = f"""{TempfileUtils.temp_datasource_path}/{date_path}/{service_name}/{
+            token}"""
+        folder_path = Path(temp_dir)
+        folder_path.mkdir(parents=True, exist_ok=True)
+        return temp_dir
+
+    @staticmethod
+    def get_temp_file_name(service_name, token, prefix="", extension=""):
+        unique_filename = uuid.uuid1().hex
+
+        if prefix:
+            if prefix.endswith("_"):
+                unique_filename = f"{prefix}{unique_filename}"
+            else:
+                unique_filename = f"{prefix}_{unique_filename}"
+
+        if extension:
+            if extension.startswith("."):
+                unique_filename = f"{unique_filename}{extension}"
+            else:
+                unique_filename = f"{unique_filename}.{extension}"
+
+        temp_path = TempfileUtils.get_temp_path(service_name, token)
+        full_path = Path(temp_path) / unique_filename
+        return full_path

pam/templates/buildcmd/pamb

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+CMD=$1
+APP_VERSION=$2
+IMAGE=$3
+
+function print_howto() {
+    echo "How to use : pamb CMD APP_VERSION TIMESTAMP NAMESPACE OTHER_PARAM1 OTHER_PARAM2 (eg. pamb build_pam 6.0 201906011030 stg)"
+}
+
+if [ "$CMD" = "" ]; then
+    echo -e "CMD cannot be blank"
+    print_howto
+    exit 1
+fi
+
+if [ "$APP_VERSION" = "" ]; then
+    echo -e "APP_VERSION cannot be blank"
+    print_howto
+    exit 1
+fi
+
+if [ "$IMAGE" = "" ]; then
+    echo -e "IMAGE cannot be blank"
+    print_howto
+    exit 1
+fi
+
+source ./pamb-base.sh
+
+eval ${CMD}

pam/templates/buildcmd/pamb-base.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+function commit_version() {
+    local IMAGE=$1
+    local VERSION=$2
+    echo "docker push image : 3dsinteractive/$IMAGE:$VERSION"
+    docker login -u $DOCKER_USER -p $DOCKER_PASS
+    docker push 3dsinteractive/$IMAGE:$VERSION
+}
+
+function prepare_project() {
+    git config --global url.ssh://git@bitbucket.org/.insteadOf https://bitbucket.org/
+}
+
+function build_plugin() {
+    prepare_project
+
+    docker build -f Dockerfile -t 3dsinteractive/$IMAGE:$APP_VERSION .
+    commit_version $IMAGE $APP_VERSION
+}

pam/templates/docker/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12-slim
+
+COPY . /app
+
+WORKDIR /app
+
+ENV DATA_FOLDER=/app/data
+RUN mkdir -p $DATA_FOLDER
+
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
+EXPOSE 8000
+
+ENV PYTHONUNBUFFERED=1
+
+CMD ["gunicorn", "-w", "1", "-b", "0.0.0.0:8000", "main:app"]

pam/templates/init/main.tmpl

@@ -0,0 +1,11 @@
+import os
+from flask import Flask
+from pam.server import Server
+
+
+app = Flask(__name__)
+
+pluginServer = Server(app)
+
+if __name__ == '__main__':
+    pluginServer.run(host='0.0.0.0', port=8000)

pam/templates/service/ServiceClass.tmpl

@@ -0,0 +1,48 @@
+import pandas as pd
+from pam.service import Service
+from pam.models.request_command import RequestCommand
+from pam.utils import log
+from functions import *
+
+
+class #CLASS_NAME#(Service):
+
+    def __init__(self, task_manager, req):
+        super().__init__(task_manager, req)
+
+    def on_start(self):
+        log("on_start")
+        self._request_data()
+
+    def on_data_input(self, req: RequestCommand):
+        log(f"on_data_input req.is_end = {req.is_end}")
+
+        dataframe = self.__process_data(req.input_files)
+        self._upload_result(dataframe)
+
+        if not req.is_end:
+            self._request_data(req.next)
+        else:
+            report_df = create_report()
+            report_name = "#CLASS_NAME#_report1"
+            self._upload_report(report_name, report_df)
+                
+            self._exit()
+
+    def __process_data(self, input_files):
+        file_path = input_files[0]
+        df = load_csv_file(file_path)
+        df['finished'] = True
+        return df
+
+    def __create_report():
+        pass
+
+    def on_destroy(self):
+        log("on_destroy")
+
+    def on_terminate(self):
+        log("on_terminate")
+
+    def get_status(self):
+        return ""

pam/templates/service/functions.tmpl

@@ -0,0 +1,22 @@
+import pandas as pd
+
+def load_csv_file(csv_file):
+    dtype_spec = {
+        'id': 'str'
+    }
+    df = pd.read_csv(csv_file, dtype=dtype_spec, parse_dates=['created_date'])
+    return df
+
+def create_report():
+    report = {
+        "type": "TABLE",
+        "headers": ["test",
+                    "column1",
+                    "column2"
+                    ],
+        "rows": [
+            ["test1", 1, 2],
+            ["test2", 3, 4]
+        ]
+    }
+    return report

pam/templates/service/service.test.tmpl

@@ -0,0 +1,83 @@
+
+from typing import List, Tuple
+from #CLASS_NAME#_service import #CLASS_NAME#
+from pam.tester_task import TesterTask
+from pam.models.request_command import RequestCommand
+from faker import Faker
+import pandas as pd
+import os
+import shutil
+import uuid
+from pam.temp_file_utils import TempfileUtils
+
+def __create_mock_csv(page):
+    fake = Faker()
+    data = {
+        "id": range(1, 11),  # 10 rows
+        "name": [fake.name() for _ in range(10)],
+        "age": [fake.random_int(min=18, max=65) for _ in range(10)],
+        "email": [fake.email() for _ in range(10)],
+        "created_date": [fake.date_time_this_year() for _ in range(10)]
+    }
+    file_name = f"{mock_dir}/mock_data_faker_{page}.csv"
+    df = pd.DataFrame(data)
+    df.to_csv(file_name, index=False)
+    return file_name
+        
+def can_convert_to_int(x):
+    try:
+        int(x)
+        return True
+    except (ValueError, TypeError):
+        return False
+    
+def on_request_data(page) -> Tuple[List[str], bool, str]:
+    page_num = 1
+    if can_convert_to_int(page):
+        page_num = int(page)
+
+    next_page = str(page_num + 1)
+    is_end = False
+    if page_num == MAXPAGE:
+        is_end = True
+        next_page = ""
+
+    mock_csv = __create_mock_csv(page_num)
+    
+    return [mock_csv], is_end, next_page
+
+
+def on_upload_report(report_file):
+    print(f"Uploaded Report {report_file}")
+
+
+def on_upload_result(result_file):
+    print(f"Uploaded {result_file}")
+
+
+
+#---- Constants ----
+MAXPAGE=3
+mock_dir = "./TEMP_TEST_DATA"
+TempfileUtils.temp_base_path = f"{mock_dir}/app/data"
+TempfileUtils.temp_datasource_path = f"{mock_dir}/app/data/data_sources"
+
+
+#---- Create Mock Dir ----
+if os.path.exists(mock_dir):
+    shutil.rmtree(mock_dir)
+os.makedirs(mock_dir)
+
+#---- Create Tester Task ----
+testerTask = TesterTask()
+testerTask.set_on_upload_result(on_upload_result)
+testerTask.set_on_request_data(on_request_data)
+testerTask.set_on_upload_report(on_upload_report)
+
+#---- Create Request Command ----
+token=unique_id = uuid.uuid4()
+req = RequestCommand(token, "start", "-", "-", None, None, None, '#CLASS_NAME#')
+#CLASS_NAME# = #CLASS_NAME#(testerTask, req)
+
+#---- Start Test ----
+testerTask.test_service(#CLASS_NAME#)

pam/templates/service/service.yaml

@@ -0,0 +1,2 @@
+endpoint: /#CLASS_NAME#
+class: #CLASS_NAME#

pam/tester_task.py

@@ -0,0 +1,60 @@
+from typing import Callable, Tuple, List
+from pam.models.request_command import RequestCommand
+from pam.interface_task_manager import ITaskManager
+from pam.service import Service
+
+
+RequestDataCallbackType = Callable[[str], Tuple[List[str], bool, str]]
+
+UploadResultCallbackType = Callable[[str], None]
+
+UploadReportCallbackType = Callable[[str], None]
+
+
+class TesterTask(ITaskManager):
+
+    def __init__(self):
+        self.request_data_callback: RequestDataCallbackType = None
+        self.upload_result_calllback: UploadResultCallbackType = None
+        self.upload_report_calllback: UploadReportCallbackType = None
+
+    def set_on_request_data(self, request_data_callback: RequestDataCallbackType):
+        self.request_data_callback = request_data_callback
+
+    def set_on_upload_result(self, upload_result_calllback: UploadResultCallbackType):
+        self.upload_result_calllback = upload_result_calllback
+
+    def set_on_upload_report(self, upload_report_calllback: UploadReportCallbackType):
+        self.upload_report_calllback = upload_report_calllback
+
+    def test_service(self, service):
+        service.on_start()
+
+    # -----
+
+    def on_dataset_input(self, req: RequestCommand):
+        pass
+
+    def start_service(self, service_class, req: RequestCommand, service_name):
+        pass
+
+    def terminate_service(self, token):
+        pass
+
+    def service_exit(self, service_class):
+        print(f"{service_class.request.service_name} Exit.")
+
+    def service_request_data(self, service: Service, page):
+        if self.request_data_callback is not None:
+            files, is_end, next_page = self.request_data_callback(page)
+            req = RequestCommand(service.request.token, "dataset", "",
+                                 "", is_end, next_page, files, service.request.service_name)
+            service.on_data_input(req)
+
+    def service_upload_result(self, service: Service, file_path):
+        if self.upload_result_calllback is not None:
+            self.upload_result_calllback(file_path)
+
+    def service_upload_report(self, service: Service, file_path):
+        if self.upload_report_calllback is not None:
+            self.upload_report_calllback(file_path)

pam/utils.py

@@ -0,0 +1,38 @@
+import os
+import tempfile
+from datetime import datetime
+import re
+import pytz
+
+
+def get_adapter_id(url: str) -> str:
+    pattern = r'/api/dataadapter/(\w+)/response'
+    match = re.search(pattern, url)
+    if match:
+        return match.group(1)
+    return ""
+
+
+def deep_convert_numbers_to_strings(data):
+    if isinstance(data, dict):
+        return {key: deep_convert_numbers_to_strings(
+            value) for key, value in data.items()}
+
+    if isinstance(data, list):
+        return [deep_convert_numbers_to_strings(item) for item in data]
+
+    if isinstance(data, tuple):
+        return tuple(deep_convert_numbers_to_strings(item) for item in data)
+
+    if isinstance(data, (int, float)):
+        return str(data)
+
+    return data
+
+
+def log(msg: str):
+    utc_dt = datetime.now(pytz.utc)  # Get current time in UTC
+    tz = pytz.timezone('Asia/Bangkok')
+    bangkok_time = utc_dt.astimezone(tz)  # Convert UTC time to Bangkok time
+    date_str = bangkok_time.strftime("%m/%d/%Y %H:%M:%S")
+    print(f"[{date_str}]: {msg}")

pam_python-0.1.0.dist-info/LICENSE.txt

@@ -0,0 +1,17 @@
+This software is provided as part of the agreement between 3dsinteractive,co.ltd ("the Provider") and its customers ("the Licensee") under the following conditions:
+
+1. **Usage Rights for Active Customers**:
+   - The Licensee may use, modify, and distribute this software exclusively for their internal operations in connection with the services provided by the Provider.
+   - No disclosure of source code is required while the Licensee maintains active status as a customer.
+
+2. **Termination of Customer Status**:
+   - Upon termination of the agreement between the Provider and the Licensee, the Licensee must either:
+     a. Cease using and delete all copies of this software, or
+     b. Open-source any derived works based on this software under an OSI-approved license.
+
+3. **Restrictions**:
+   - Redistribution of this software to third parties not under the Licensee's operational control is strictly prohibited.
+   - The Licensee may not use this software for competitive purposes against the Provider.
+
+4. **Liability**:
+   - The Provider provides this software "as is" without warranty of any kind, either expressed or implied.

pam_python-0.1.0.dist-info/METADATA

@@ -0,0 +1,29 @@
+Metadata-Version: 2.1
+Name: pam-python
+Version: 0.1.0
+Summary: Pam Python Library
+Home-page: https://github.com/heart/pam-python
+Author: Narongrit Kanhanoi
+Author-email: narongrit@pams.ai
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Customer Service
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: setuptools>=70.0.0
+Requires-Dist: Flask>=3.0.2
+Requires-Dist: aiohttp>=3.11.11
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: Faker>=33.1.0
+Requires-Dist: dask>=2024.12.1
+Requires-Dist: dask-expr>=1.1.21
+
+## License
+
+This library is distributed under a custom license. Active customers of [Your Company Name] may use it as per the terms stated in the `LICENSE.txt` file. For details, contact contact@3dsinteractive.com.

pam_python-0.1.0.dist-info/RECORD

@@ -0,0 +1,26 @@
+pam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pam/api.py,sha256=pNY1qTSTalCk8nzXRZ845gXGZwpK2iNwOtjZnEZdEDY,1472
+pam/cli.py,sha256=dqfGXMJGBqt1HJ-ZbxHLndPa9hV_UaHgu0SvxwFAZrQ,3671
+pam/interface_task_manager.py,sha256=6DGAUcf6IxNIYIZZcFRjZh6NynSm5BWZgrJP0GdCrdQ,717
+pam/server.py,sha256=hvp-ZQcp2zJTFxOeVLEQg3f9aZgBLwTTSgfeRV94J6A,5374
+pam/service.py,sha256=oavcz_ggkKTk7RgvGNwwpbHw75nqg2b5fTxsMMKkKwE,2803
+pam/task_manager.py,sha256=6ChMLTLp9GCxKUcJAZzHtb9l-5K83mc_3XRwtxD-Esk,5011
+pam/temp_file_utils.py,sha256=pvI8klHQqnH3-GL_7vGPHKtvZRyEC1PBGqbxrqwi05Y,2434
+pam/tester_task.py,sha256=S-fxXMH5FMaeJVv_FAxHiJCFb3cStSQoRrRqxK8Yutk,2191
+pam/utils.py,sha256=nOndtbbdlHx5vfUZJpO-qR59eWvvF3xoCBOQSqKckiM,1029
+pam/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pam/models/request_command.py,sha256=KnARL5sVbIr_xUcF0EMMxCDHQRX57M2gUcRLPKm2Vjs,4239
+pam/templates/buildcmd/pamb,sha256=58GCN59ysvHGNhPvTD3nkF4aLg-kM1WfWmH7CL3eQVo,530
+pam/templates/buildcmd/pamb-base.sh,sha256=Q0DkbeHtPx33PIjc68EeI7ZIwBOLlwcSmPhpxALXxVY,518
+pam/templates/docker/Dockerfile,sha256=d266YB4XYRJEM3mtU9uSQcL_rsCZojoA04fW8OqovPU,268
+pam/templates/init/main.tmpl,sha256=v0n0H4YNrRPPPZlbzdL2vpLwQgCMgJQ6Cm7nR6oo4qU,192
+pam/templates/service/ServiceClass.tmpl,sha256=M30LOdAIBfnLBLhBO--3DWzUz1EWwtcRHS-I-vUBvZE,1188
+pam/templates/service/functions.tmpl,sha256=RMG8_PsKyhHgBMeXBfCavRlUW00rqNoo6QKfG3pZgqU,470
+pam/templates/service/service.test.tmpl,sha256=YmdEnVMmdpwM8iT1nXF1fSKOjg0XBWXfbXRwcecxi0s,2198
+pam/templates/service/service.yaml,sha256=S9r6MeysjlBZVr-6OykrwFO5RJqrhsoLr6R8FQtD8gg,44
+pam_python-0.1.0.dist-info/LICENSE.txt,sha256=yeHD2-gzrkkET2ZkMe22WyUGpDjlwTrqA4POb0FoZi4,1138
+pam_python-0.1.0.dist-info/METADATA,sha256=nnkU4wLWA0kRda746Ytv0_s5AHIdYbUvMYsT60nHUEg,1102
+pam_python-0.1.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+pam_python-0.1.0.dist-info/entry_points.txt,sha256=xqYpsIqvNlO0j8elZc02bDwR-12dLK9FU2B1ACsLYxw,37
+pam_python-0.1.0.dist-info/top_level.txt,sha256=0EOjbyc3hQyzjhn6iyMgsEseqA66Xz0p27iBN7G7W1w,4
+pam_python-0.1.0.dist-info/RECORD,,

pam_python-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.6.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pam_python-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pam = pam.cli:main

pam_python-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pam