pam-python 0.1.0__tar.gz

pam_python-0.1.0/LICENSE.txt

@@ -0,0 +1,17 @@
+This software is provided as part of the agreement between 3dsinteractive,co.ltd ("the Provider") and its customers ("the Licensee") under the following conditions:
+
+1. **Usage Rights for Active Customers**:
+   - The Licensee may use, modify, and distribute this software exclusively for their internal operations in connection with the services provided by the Provider.
+   - No disclosure of source code is required while the Licensee maintains active status as a customer.
+
+2. **Termination of Customer Status**:
+   - Upon termination of the agreement between the Provider and the Licensee, the Licensee must either:
+     a. Cease using and delete all copies of this software, or
+     b. Open-source any derived works based on this software under an OSI-approved license.
+
+3. **Restrictions**:
+   - Redistribution of this software to third parties not under the Licensee's operational control is strictly prohibited.
+   - The Licensee may not use this software for competitive purposes against the Provider.
+
+4. **Liability**:
+   - The Provider provides this software "as is" without warranty of any kind, either expressed or implied.

pam_python-0.1.0/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.1
+Name: pam-python
+Version: 0.1.0
+Summary: Pam Python Library
+Home-page: https://github.com/heart/pam-python
+Author: Narongrit Kanhanoi
+Author-email: narongrit@pams.ai
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Customer Service
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: setuptools>=70.0.0
+Requires-Dist: Flask>=3.0.2
+Requires-Dist: aiohttp>=3.11.11
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: Faker>=33.1.0
+Requires-Dist: dask>=2024.12.1
+Requires-Dist: dask-expr>=1.1.21
+
+## License
+
+This library is distributed under a custom license. Active customers of [Your Company Name] may use it as per the terms stated in the `LICENSE.txt` file. For details, contact contact@3dsinteractive.com.

pam_python-0.1.0/README.md

@@ -0,0 +1,3 @@
+## License
+
+This library is distributed under a custom license. Active customers of [Your Company Name] may use it as per the terms stated in the `LICENSE.txt` file. For details, contact contact@3dsinteractive.com.

pam_python-0.1.0/pam/api.py

@@ -0,0 +1,42 @@
+import requests
+from pam.utils import log
+
+
+class API:
+
+    def http_post(self, url, data):
+        """
+        Sends an HTTP POST request to the specified URL with the given data as JSON.
+
+        :param url: The URL to send the POST request to.
+        :param data: A dictionary to be used as the JSON body of the POST request.
+        :return: The response from the server.
+        """
+        headers = {
+            'Content-Type': 'application/json'}
+        try:
+            response = requests.post(
+                url, json=data, timeout=30, headers=headers)
+            response.raise_for_status()  # Raise an exception for HTTP errors
+            return response
+        except requests.RequestException as e:
+            log(f"HTTP POST request failed: {e}")
+            return None
+
+    def http_upload(self, url, file_path):
+        """
+        Uploads a file to the specified URL.
+
+        :param url: The URL to upload the file to.
+        :param file_path: The path to the file to be uploaded.
+        :return: The response from the server.
+        """
+        try:
+            with open(file_path, 'rb') as file:
+                files = {'file': file}
+                response = requests.post(url, files=files, timeout=300)
+                response.raise_for_status()  # Raise an exception for HTTP errors
+                return response
+        except requests.RequestException as e:
+            log(f"File upload failed: {e}")
+            return None

pam_python-0.1.0/pam/cli.py

@@ -0,0 +1,107 @@
+import os
+import sys
+import pkg_resources 
+import shutil
+import glob
+import subprocess
+
+def main():
+    args = sys.argv[1:]
+    if len(args) == 0:
+        print("Usage: pam <command> [args]")
+        return
+    
+    cmd = args[0]
+
+    if cmd == "init":
+        init_project()
+    elif cmd == "new":
+        type = args[1]
+        if type == "service":
+            name = args[2]
+            create_service(name)
+    elif cmd == "test":
+        if len(args) < 2:
+            print("Usage: pam test <modulename>")
+            return
+        module_name = args[1]
+        test_module(module_name)
+    else:
+        print(f"Unknown command: {args.command}")
+
+def test_module(module_name):
+    """Run all test files matching '*.test.py' in the specified module directory."""
+    # Locate all test files in the module directory
+    test_files = glob.glob(f"{module_name}/*.test.py")
+    
+    if not test_files:
+        print(f"Error: No test files found in module '{module_name}' (expected '*.test.py').")
+        return
+    
+    print(f"Found {len(test_files)} test file(s) in '{module_name}':")
+    for test_file in test_files:
+        print(f" - {test_file}")
+    
+    # Run each test file
+    for test_file in test_files:
+        print(f"\nRunning tests in {test_file}...")
+        result = subprocess.run([sys.executable, test_file], capture_output=True, text=True)
+
+        print("\nTest Output:")
+        print(result.stdout)
+        if result.stderr:
+            print("\nErrors:")
+            print(result.stderr)
+        print("-" * 40)  # Separator for clarity
+
+def cpy(src, dest):
+    template_dir = pkg_resources.resource_filename("pam", "templates")
+    src_file = os.path.join(template_dir, src)
+    shutil.copy(src_file, dest)
+
+def replace_template_content(service_name, file_name):
+    file_path = os.path.join(service_name, file_name)
+    with open(file_path, 'r+') as file:
+        filedata = file.read()
+        updated_data = filedata.replace('#CLASS_NAME#', service_name)
+        file.seek(0)  # Move the file pointer to the beginning of the file
+        file.write(updated_data)
+        file.truncate()  # Remove any leftover content after the replacement
+
+def create_service(name):
+    if os.path.exists(name):
+        response = input(f"Service {name} already exists. Do you want to overwrite it? (y/N): ").strip().lower()
+        if response == 'y':
+            shutil.rmtree(name)
+        else:
+            print("Cancelled.")
+            return
+    
+    os.mkdir(name)
+    open(os.path.join(name, "__init__.py"), 'a', encoding='utf-8').close()
+    cpy("service/ServiceClass.tmpl", os.path.join(name, name+"_service.py") )
+    cpy("service/service.yaml", os.path.join(name, "service.yaml") )
+    cpy("service/functions.tmpl", os.path.join(name, "functions.py") )
+    cpy("service/service.test.tmpl", os.path.join(name, name+".test.py") )
+
+    
+    replace_template_content(name, name+"_service.py")
+    replace_template_content(name, "service.yaml")
+    replace_template_content(name, name+".test.py")
+
+    print(f"Service {name} created.")
+    print(f"Run `pam test {name}` to run tests for the service.")
+
+def init_project():
+    cpy("init/main.tmpl", "main.py")
+    cpy("docker/Dockerfile", "Dockerfile")
+    cpy("buildcmd/pamb", "pamb")
+    cpy("buildcmd/pamb-base.sh", "pamb-base.sh")
+    if not os.path.exists("requirements.txt"):
+        open("requirements.txt", 'a', encoding='utf-8').close()
+    
+    with open("requirements.txt", "w", encoding='utf-8') as f:
+        subprocess.run(["pip", "freeze"], stdout=f, check=True)
+
+    print("--- Welcome to PAM ---\n")
+    print("To create a new servive run\n`pam new service <service_name>`\n\n")

pam_python-0.1.0/pam/interface_task_manager.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from pam.models.request_command import RequestCommand
+from pam.service import Service
+
+
+class ITaskManager(ABC):
+
+    @abstractmethod
+    def on_dataset_input(self, req: RequestCommand):
+        pass
+
+    @abstractmethod
+    def start_service(self, service_class, req: RequestCommand, service_name):
+        pass
+
+    @abstractmethod
+    def terminate_service(self, token):
+        pass
+
+    @abstractmethod
+    def service_request_data(self, service: Service, page):
+        pass
+
+    @abstractmethod
+    def service_upload_result(self, service: Service, file_path):
+        pass
+
+    @abstractmethod
+    def service_upload_report(self, service: Service, file_path):
+        pass

pam_python-0.1.0/pam/models/request_command.py

@@ -0,0 +1,138 @@
+
+"""
+RequestCommand is object that Wrap the http request to object that simple to access the parameters
+"""
+
+import os
+from dataclasses import dataclass
+import zipfile
+from pathlib import Path
+from pam.utils import log
+from pam.temp_file_utils import TempfileUtils
+
+
+@dataclass
+class RequestCommand:
+    """
+    RequestCommand is object that Wrap the http request
+    """
+
+    token: str
+    cmd: str
+    data_request_api: str
+    response_api: str
+    is_end: bool
+    next: str
+    input_files: list[str]
+    service_name: str
+
+    def is_start_command(self):
+        """Determine is this command is start command"""
+        return self.cmd == "start"
+
+    def is_dataset_command(self):
+        """Determine is this command is dataset command"""
+        return self.cmd == "dataset"
+
+    @staticmethod
+    def __safe_str_to_bool(value):
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            if value.lower() in ['true', '1', 'yes']:
+                return True
+            if value.lower() in ['false', '0', 'no']:
+                return False
+        return False
+
+    @staticmethod
+    def __extract_data_from_request(request, keys):
+        result = {}
+
+        if request.content_type == 'application/json':
+            data = request.get_json()
+            for key in keys:
+                if key in data:
+                    result[key] = data[key]
+                else:
+                    result[key] = ""
+        elif request.content_type.startswith('multipart/form-data'):
+            for key in keys:
+                data_str = request.form.get(key, '')
+                if data_str:
+                    result[key] = data_str
+                else:
+                    result[key] = ""
+
+        return result
+
+    @staticmethod
+    def __allowed_file(filename):
+        return '.' in filename and filename.rsplit(
+            '.', 1)[1].lower() in {'zip'}
+
+    @staticmethod
+    def __extract_input_file(zip_file, service_name, token):
+
+        if zip_file.filename == '':
+            log("handle_file_upload 'No selected file'")
+            return ([], "No selected file")
+
+        if not RequestCommand.__allowed_file(zip_file.filename):
+            log("handle_file_upload 'Invalid file type'")
+            return ([], "Invalid file type")
+
+        zip_file_dir = TempfileUtils.get_temp_file_name(
+            service_name, token, "dataset_")
+
+        zip_file_name = f"{zip_file_dir}.zip"
+        zip_file.save(zip_file_name)
+
+        extract_dir = Path(zip_file_dir)
+        extract_dir.mkdir(parents=True, exist_ok=True)
+
+        with zipfile.ZipFile(zip_file_name, 'r') as zip_ref:
+
+            zip_ref.extractall(zip_file_dir)
+
+            if os.path.exists(f"{extract_dir}/__MACOSX"):
+                os.rmdir(f"{extract_dir}/__MACOSX")
+            if os.path.exists(f"{extract_dir}/.DS_Store"):
+                os.rmdir(f"{extract_dir}/.DS_Store")
+
+        input_files = sorted([os.path.join(extract_dir, name)
+                              for name in os.listdir(extract_dir)])
+
+        log(f"Extract dir: {extract_dir}")
+        log(f"File save to: {input_files}")
+
+        return (input_files, "")
+
+    @staticmethod
+    def parse(request, service_name):
+        """Create RequestCommand object from http request"""
+        params = RequestCommand.__extract_data_from_request(
+            request, ["cmd", "token", "data", "response", "is_end", "next"])
+
+        token = params["token"]
+        cmd = params["cmd"]
+        data = params["data"]
+        response = params["response"]
+        is_end = RequestCommand.__safe_str_to_bool(params["is_end"])
+        next_page = params["next"]
+        input_files = []
+        error_message = ""
+
+        if not token:
+            error_message = 'The `token` parameter is required.'
+        else:
+            if cmd == "dataset":
+                zip_file = request.files['file']
+                (files, error) = RequestCommand.__extract_input_file(
+                    zip_file, service_name, token)
+                error_message = error
+                input_files = files
+
+        return RequestCommand(
+            token, cmd, data, response, is_end, next_page, input_files, service_name
+        ), error_message

pam_python-0.1.0/pam/server.py

@@ -0,0 +1,170 @@
+"""
+Data Plugin Server
+"""
+
+import os
+import importlib
+import yaml
+from setuptools import find_packages
+from flask import request, make_response, Flask, jsonify
+from pam.utils import log
+from pam.models.request_command import RequestCommand
+from pam.task_manager import TaskManager
+from pam.temp_file_utils import TempfileUtils
+
+
+class Server:
+    """
+    Data Plugin Server
+    """
+
+    app: Flask
+    servicePool = {}
+
+    def __init__(self, app):
+        self.task_manager = TaskManager(self)
+        self.app = app
+        self.register_service()
+        self.task_manager.start_service_monitoring_schedul()
+
+        @app.route('/', methods=['GET'])
+        def home():
+            return self.response_ok({'message': "Hello, Data plugin v2."})
+
+        @app.route('/service/<service_name>', methods=['POST'])
+        def run(service_name):
+            TempfileUtils.clean_temp()
+
+            req, error_request = RequestCommand.parse(request, service_name)
+
+            if error_request != "":
+                response = {
+                    'message': error_request
+                }
+                return self.response_error(response, 400)
+
+            response, code = self.handle_service_cmd(service_name, req)
+            return self.response_error(response, code)
+
+        @app.route('/tasks', methods=['GET'])
+        def tasks():
+            return self.response_ok({'message': "OK"})
+
+        @app.route('/status/<service_name>', methods=['GET'])
+        def ping(service_name):
+            # TODO: check the service status
+            return self.response_ok({'message': f"OK {service_name}"})
+
+    def get_service_class(self, service_name):
+        """
+        Get service class from service name
+        """
+        if service_name in self.servicePool:
+            return self.servicePool[service_name]
+
+        return None
+
+    def handle_service_cmd(self, service_name, req: RequestCommand):
+        """
+        Handle all cmd type
+        """
+        service_class = self.get_service_class(service_name)
+
+        if service_class is not None:
+            if req.is_start_command():
+                log(f"Request on start {service_name} token={req.token}")
+                return self.on_start(service_class, req, service_name)
+
+            if req.is_dataset_command():
+
+                log(f"""Request on dataset {service_name} token={
+                    req.token} files={req.input_files}""")
+
+                return self.on_dataset(req)
+
+        response = {'message': f'Service `{service_name}` Notfound. '}
+        return response, 404
+
+    def on_start(self, service_class, req, service_name):
+        """
+        start a service
+        """
+        log(f"{service_name} -> on_start")
+        self.task_manager.start_service(service_class, req, service_name)
+        return {'acknowledge': True}, 200
+
+    def on_dataset(self, req):
+        """
+        task_manager handle cmd=dataset
+        """
+        log(f"{req.service_name} -> on_dataset")
+        self.task_manager.on_dataset_input(req)
+        return {'acknowledge': True}, 200
+
+    def on_register_service(self, service, end_point):
+        """
+        register a service
+        """
+        log(f"register service \"{service}\", endpoint: http://localhost/service/{end_point}")
+        self.servicePool[end_point] = service
+
+    def run(self, host='0.0.0.0', port=8000):
+        """
+        run flask app
+        """
+        self.app.run(host=host, port=port)
+
+    def response_error(self, json, code):
+        """
+        create error response with custom http status code
+        """
+        data = jsonify(json)
+        return make_response(data, code)
+
+    def response_ok(self, json, code=200):
+        """
+        create response object with 200OK
+        """
+        data = jsonify(json)
+        return make_response(data, code)
+
+    def get_service_config(self, package):
+        """
+        load the service config from service.yml
+        """
+        config_path_1 = f'{package}/service.yaml'
+        config_path_2 = f'{package}/service.yml'
+        if os.path.exists(config_path_1) and os.path.isfile(config_path_1):
+            with open(config_path_1, 'r', encoding='utf-8') as file:
+                config = yaml.safe_load(file)
+                return config
+        elif os.path.exists(config_path_2) and os.path.isfile(config_path_2):
+            with open(config_path_2, 'r', encoding='utf-8') as file:
+                config = yaml.safe_load(file)
+                return config
+        return None
+
+    def register_service(self):
+        """
+        scan project to register all plugin services
+        """
+        packages = find_packages()
+        for package in packages:
+            config = self.get_service_config(package)
+            if config is not None:
+                endpoint = config['endpoint']
+                class_name = config['class']
+                module = importlib.import_module(
+                    f'{package}.{class_name}')
+
+                try:
+                    class_ = getattr(module, class_name)
+                except AttributeError:
+                    log(
+                        f"Error: Class '{class_name}' not found in module '{module}'.")
+                    class_ = None
+
+                if endpoint.startswith("/"):
+                    endpoint = endpoint[1:]
+
+                self.on_register_service(class_, endpoint)

pam_python-0.1.0/pam/service.py

@@ -0,0 +1,101 @@
+"""
+Service base class
+"""
+
+import os
+import tempfile
+import uuid
+from abc import abstractmethod
+import json
+from typing import TYPE_CHECKING
+import pandas as pd
+import dask.dataframe as dd
+from pam.utils import log, deep_convert_numbers_to_strings, get_adapter_id
+from pam.models.request_command import RequestCommand
+from pam.temp_file_utils import TempfileUtils
+
+if TYPE_CHECKING:
+    from pam.task_manager import TaskManager
+
+
+class Service:
+    """
+    Service base class
+    """
+
+    # pylint: disable=too-many-instance-attributes
+    def __init__(self, task_manager: 'TaskManager', req: RequestCommand):
+        self.request = req
+        self.task_manager = task_manager
+
+    # == REQUEST DATA ===
+    def _request_data(self, page=""):
+        log(f"{self.request.service_name}: Request data page={page}")
+        self.task_manager.service_request_data(self, page)
+
+    # == UPLOAD RESULT ===
+    def _upload_result(self, df):
+        if isinstance(df, dd.DataFrame):
+            df = df.compute()
+        elif not isinstance(df, pd.DataFrame):
+            raise ValueError("The input must be a Pandas or Dask DataFrame")
+
+        tmp_file_name = TempfileUtils.get_temp_file_name(
+            self.request.service_name, self.request.token, "result_", ".csv")
+
+        df.to_csv(tmp_file_name, index=False)
+
+        log(f"{self.request.service_name}: Upload Result File {tmp_file_name}")
+        self.task_manager.service_upload_result(self, tmp_file_name)
+
+        return tmp_file_name
+
+    # == UPLOAD REPORT ===
+
+    def _upload_report(self, report_name: str, report_json: str):
+        adapter_id = get_adapter_id(self.request.response_api)
+        service_name = self.request.service_name
+        token = self.request.token
+
+        # important need to recursive and convert numbers to str
+        report_json = deep_convert_numbers_to_strings(
+            report_json)
+
+        json_string = json.dumps(report_json)
+
+        report_name = f"data_{report_name}_no_index"
+
+        df = pd.DataFrame({
+            'customer': [adapter_id],
+            report_name: [json_string]
+        })
+
+        report_csv_path = TempfileUtils.get_temp_file_name(
+            service_name, token, f"report_{report_name}_", '.csv')
+
+        df.to_csv(report_csv_path, index=False)
+        self.task_manager.service_upload_report(self, report_csv_path)
+    # =========
+
+    def _exit(self):
+        self.task_manager.service_exit(self)
+
+    @abstractmethod
+    def on_start(self):
+        """on sstart"""
+
+    @abstractmethod
+    def on_data_input(self, req):
+        """on_data_input"""
+
+    @abstractmethod
+    def on_destroy(self):
+        """on_destroy"""
+
+    @abstractmethod
+    def on_terminate(self):
+        """on_terminate"""
+
+    @abstractmethod
+    def get_status(self):
+        """get_status"""