pam-python 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

pam/__init__.py

@@ -1,3 +1,3 @@
 """PAM Data Plugin framework."""
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"

pam/cli.py

@@ -1,168 +1,235 @@
+import importlib.resources as resources
+from importlib import metadata
+import json
 import os
+from pathlib import Path
 import re
 import shutil
+import subprocess
 import sys
-import importlib.resources as resources
-from importlib import metadata
+import tempfile
+
+
+PROJECT_MARKER = ".pam-project"
+PACKAGE_MANAGERS = {"1": "uv", "2": "pip"}
+DATAFRAME_ENGINES = {"1": "polars", "2": "pandas"}
+
+COMMON_PROJECT_TEMPLATES = {
+    "init/main.tmpl": "main.py",
+    "buildcmd/pamb": "pamb",
+    "buildcmd/pamb-base.sh": "pamb-base.sh",
+    "init/pylintrc.tmpl": ".pylintrc",
+    "init/REPORTS.md": "REPORTS.md",
+    "init/AGENT.md": "AGENT.md",
+    "init/gitignore.tmpl": ".gitignore",
+    "init/dockerignore.tmpl": ".dockerignore",
+}
+
+PACKAGE_MANAGER_TEMPLATES = {
+    "pip": {
+        "project/pip/Dockerfile": "Dockerfile",
+        "project/pip/requirements.txt": "requirements.txt",
+        "init/run_unit_test.sh": "run_unit_test.sh",
+        "init/run_unit_test.bat": "run_unit_test.bat",
+        "init/run_unit_test.ps1": "run_unit_test.ps1",
+    },
+    "uv": {
+        "project/uv/Dockerfile": "Dockerfile",
+        "project/uv/pyproject.toml": "pyproject.toml",
+        "project/uv/python-version": ".python-version",
+    },
+}
+
+SERVICE_TEMPLATES = {
+    "polars": {
+        "service/polars/service_class.tmpl": "#CLASS_NAME#.py",
+        "service/polars/functions.tmpl": "functions.py",
+        "service/polars/service.test.tmpl": "test_#MODULE_NAME#.py",
+    },
+    "pandas": {
+        "service/pandas/service_class.tmpl": "#CLASS_NAME#.py",
+        "service/pandas/functions.tmpl": "functions.py",
+        "service/pandas/service.test.tmpl": "test_#MODULE_NAME#.py",
+    },
+}
+
+ENGINE_DEPENDENCIES = {
+    "polars": ["polars>=1.0.0"],
+    "pandas": ["pandas>=2.2.3", "pyarrow>=19.0.1"],
+}
 
 
 def main(argv=None):
     args = list(sys.argv[1:] if argv is None else argv)
-    if len(args) == 0:
+    if not args:
         print("Usage: pam <command> [args]")
         return
 
     cmd = args[0]
 
     if cmd == "init":
+        if len(args) != 1:
+            print("Usage: pam init")
+            return
         init_project()
     elif cmd == "new":
-        if len(args) < 2:
+        if len(args) != 3 or args[1] != "service":
             print("Usage: pam new service <service_name>")
             return
-        create_type = args[1]
-        if create_type == "service":
-            if len(args) < 3:
-                print("Usage: pam new service <service_name>")
-                return
-            name = args[2]
-            create_service(name)
-        else:
-            print(f"Unknown new target: {create_type}")
+        create_service(args[2])
     else:
         print(f"Unknown command: {cmd}")
 
 
 def to_pascal_case(input_string: str) -> str:
-    """
-    Convert a string to PascalCase.
-
-    :param input_string: The string to convert.
-    :return: The string in PascalCase.
-    """
-    # Split the string into words using non-alphanumeric characters as delimiters
-    words = re.split(r'[^A-Za-z0-9]+', input_string)
-
-    # Capitalize each word and join them
-    pascal_case = ''.join(word[:1].upper() + word[1:] for word in words if word)
-
-    return pascal_case
+    words = re.split(r"[^A-Za-z0-9]+", input_string)
+    return "".join(word[:1].upper() + word[1:] for word in words if word)
 
 
-def cpy(src, dest):
-    template_dir = resources.files("pam") / "templates"
-    src_file = os.path.join(template_dir, src)
-    shutil.copy(src_file, dest)
+def _template_path(src: str):
+    return resources.files("pam") / "templates" / src
 
 
 def read_template_text(src: str) -> str:
-    template_dir = resources.files("pam") / "templates"
-    src_file = os.path.join(template_dir, src)
-    with open(src_file, "r", encoding="utf-8") as file:
+    with open(_template_path(src), "r", encoding="utf-8") as file:
         return file.read()
 
 
-def replace_template_content(service_name, class_name, file_name):
-    file_path = os.path.join(service_name, file_name)
-    with open(file_path, 'r+', encoding='utf-8') as file:
-        filedata = file.read()
-        updated_data = filedata.replace('#CLASS_NAME#', class_name)
-        updated_data = updated_data.replace('#MODULE_NAME#', service_name)
-        file.seek(0)  # Move the file pointer to the beginning of the file
-        file.write(updated_data)
-        file.truncate()  # Remove any leftover content after the replacement
-
-
-def create_service(name):
-    if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*", name):
-        raise ValueError(
-            "service_name must start with a letter and contain only letters, "
-            "numbers, and underscores"
+def _render_template(src: str, replacements: dict[str, str]) -> str:
+    content = read_template_text(src)
+    for placeholder, value in replacements.items():
+        content = content.replace(placeholder, value)
+    return content
+
+
+def _write_rendered_template(src: str, dest: Path, replacements: dict[str, str]):
+    dest.write_text(_render_template(src, replacements), encoding="utf-8")
+    source_mode = os.stat(_template_path(src)).st_mode
+    os.chmod(dest, source_mode)
+
+
+def _select(prompt: str, options: dict[str, str], default: str) -> str:
+    if not sys.stdin.isatty():
+        raise RuntimeError(f"{prompt} requires an interactive terminal")
+    while True:
+        selection = input(f"Select [1-{len(options)}] (default: {default}): ").strip()
+        if not selection:
+            selection = default
+        if selection in options:
+            return options[selection]
+        print(f"Invalid selection: {selection}")
+
+
+def _select_package_manager() -> str:
+    print("Select package manager:")
+    print("1) uv (recommended)")
+    print("2) pip")
+    return _select("Package manager selection", PACKAGE_MANAGERS, "1")
+
+
+def _select_dataframe_engine() -> str:
+    print("Select dataframe engine:")
+    print("1) Polars (recommended)")
+    print("2) Pandas")
+    return _select("Dataframe engine selection", DATAFRAME_ENGINES, "1")
+
+
+def _project_name(directory: Path) -> str:
+    name = re.sub(r"[^a-z0-9]+", "-", directory.name.lower()).strip("-")
+    if not name:
+        name = "pam-data-plugin"
+    if name[0].isdigit():
+        name = f"pam-{name}"
+    return name
+
+
+def _assert_new_project(directory: Path):
+    known_project_files = set(COMMON_PROJECT_TEMPLATES.values())
+    for templates in PACKAGE_MANAGER_TEMPLATES.values():
+        known_project_files.update(templates.values())
+    known_project_files.update({PROJECT_MARKER, "__init__.py", "uv.lock"})
+    conflicts = [
+        name for name in sorted(known_project_files)
+        if (directory / name).exists()
+    ]
+    if conflicts:
+        raise FileExistsError(
+            "pam init only supports a new project directory; existing files: "
+            + ", ".join(conflicts)
         )
-    if os.path.exists(name):
-        response = input(
-            f"Service {name} already exists. Do you want to overwrite it? (y/N): "
-        ).strip().lower()
-        if response == 'y':
-            shutil.rmtree(name)
-        else:
-            print("Cancelled.")
-            return
-
-    os.mkdir(name)
-    open(os.path.join(name, "__init__.py"), 'a', encoding='utf-8').close()
-
-    cpy(
-        "service/service_class.tmpl",
-        os.path.join(name, to_pascal_case(name) + "Svc.py"),
-    )
 
-    cpy("service/service.yaml", os.path.join(name, "service.yaml"))
-    cpy("service/functions.tmpl", os.path.join(name, "functions.py"))
-    cpy("service/service.test.tmpl", os.path.join(name, f"test_{name}.py"))
 
-    class_name = to_pascal_case(name) + "Svc"
-    replace_template_content(name, class_name, class_name + ".py")
-    replace_template_content(name, class_name, "service.yaml")
-    replace_template_content(
-        name, class_name, f"test_{name}.py"
-    )
+def _run_uv(args: list[str], cwd: Path):
+    uv = shutil.which("uv")
+    if uv is None:
+        raise RuntimeError("uv is required for this project but was not found on PATH")
+    subprocess.run([uv, *args], cwd=cwd, check=True)
+
+
+def init_project(package_manager: str | None = None):
+    package_manager = package_manager or _select_package_manager()
+    if package_manager not in PACKAGE_MANAGER_TEMPLATES:
+        raise ValueError(f"Unsupported package manager: {package_manager}")
+
+    destination = Path.cwd()
+    _assert_new_project(destination)
+    if package_manager == "uv" and shutil.which("uv") is None:
+        raise RuntimeError("uv is required for this project but was not found on PATH")
+
+    replacements = {
+        "#PAM_VERSION#": _get_installed_pam_version(),
+        "#PROJECT_NAME#": _project_name(destination),
+    }
+    templates = {
+        **COMMON_PROJECT_TEMPLATES,
+        **PACKAGE_MANAGER_TEMPLATES[package_manager],
+    }
+
+    with tempfile.TemporaryDirectory(prefix=".pam-init-", dir=destination) as temp_dir:
+        staging = Path(temp_dir)
+        for src, dest in templates.items():
+            _write_rendered_template(src, staging / dest, replacements)
+        (staging / "__init__.py").touch()
+        (staging / PROJECT_MARKER).write_text(
+            json.dumps(
+                {
+                    "template_version": 1,
+                    "package_manager": package_manager,
+                },
+                indent=2,
+                sort_keys=True,
+            ) + "\n",
+            encoding="utf-8",
+        )
 
-    print(f"Service {name} created.")
-    print(
-        f"Run the generated run_unit_test script for your shell with `{name}` "
-        "to test the service."
-    )
+        if package_manager == "uv":
+            _run_uv(["lock"], staging)
 
+        for path in staging.iterdir():
+            os.replace(path, destination / path.name)
 
-def init_project():
-    cpy("init/main.tmpl", "main.py")
-    cpy("docker/Dockerfile", "Dockerfile")
-    cpy("buildcmd/pamb", "pamb")
-    cpy("buildcmd/pamb-base.sh", "pamb-base.sh")
-    cpy("init/pylintrc.tmpl", ".pylintrc")
-    cpy("init/gitignore.tmpl", ".gitignore")
-    cpy("init/dockerignore.tmpl", ".dockerignore")
-    cpy("init/run_unit_test.sh", "run_unit_test.sh")
-    cpy("init/run_unit_test.bat", "run_unit_test.bat")
-    cpy("init/run_unit_test.ps1", "run_unit_test.ps1")
-    cpy("init/AGENT.md", "AGENT.md")
-    cpy("init/REPORTS.md", "REPORTS.md")
-
-    requirements_path = "requirements.txt"
-    has_requirements = os.path.exists(requirements_path)
-    template_requirements = _build_project_requirements().splitlines()
-
-    mode = "overwrite"
-    if has_requirements:
-        print("requirements.txt already exists. Choose how to proceed:")
-        print("1) overwrite")
-        print("2) keep")
-        print("3) merge")
-        selection = input("Select [1-3] (default: 1): ").strip()
-        if selection == "2":
-            mode = "keep"
-        elif selection == "3":
-            mode = "merge"
-
-    if mode == "keep":
-        print("Keeping existing requirements.txt")
+    print(f"--- Welcome to PAM ({package_manager}) ---\n")
+    if package_manager == "uv":
+        print("Next:\n  uv sync\n  uv run pam new service <service_name>\n")
     else:
-        if mode == "overwrite" or not has_requirements:
-            with open(requirements_path, "w", encoding="utf-8") as f:
-                f.write(_build_project_requirements())
-        elif mode == "merge":
-            with open(requirements_path, "r", encoding="utf-8") as f:
-                existing_lines = f.read().splitlines()
-            merged_lines = _merge_requirements(existing_lines, template_requirements)
-            with open(requirements_path, "w", encoding="utf-8") as f:
-                f.write("\n".join(merged_lines) + ("\n" if merged_lines else ""))
+        print(
+            "Next:\n  python -m pip install -r requirements.txt\n"
+            "  pam new service <service_name>\n"
+        )
 
-    if not os.path.exists("__init__.py"):
-        open("__init__.py", 'a', encoding='utf-8').close()
 
-    print("--- Welcome to PAM ---\n")
-    print("To create a new service run\n`pam new service <service_name>`\n\n")
+def _load_project_config(directory: Path) -> dict:
+    marker = directory / PROJECT_MARKER
+    if not marker.is_file():
+        raise RuntimeError("This is not a PAM project created by the current `pam init`")
+    try:
+        config = json.loads(marker.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"Invalid {PROJECT_MARKER}: {exc}") from exc
+    package_manager = config.get("package_manager")
+    if package_manager not in PACKAGE_MANAGER_TEMPLATES:
+        raise RuntimeError(f"Invalid package manager in {PROJECT_MARKER}")
+    return config
 
 
 def _parse_requirement_name(line: str) -> str:
@@ -172,15 +239,103 @@ def _parse_requirement_name(line: str) -> str:
     if line.startswith("-"):
         return line
     line = line.split(";", 1)[0].strip()
-    # Normalize common requirement formats like pkg==1.2, pkg>=1.2, pkg[extra]==1.2
-    splitters = ["==", ">=", "<=", "~=", ">", "<"]
-    for splitter in splitters:
+    for splitter in ["==", ">=", "<=", "~=", ">", "<"]:
         if splitter in line:
             line = line.split(splitter, 1)[0].strip()
             break
     return line.split("[", 1)[0].strip().lower()
 
 
+def _add_pip_dependencies(dependencies: list[str], directory: Path):
+    requirements_path = directory / "requirements.txt"
+    if not requirements_path.is_file():
+        raise RuntimeError("Pip PAM project is missing requirements.txt")
+    existing = requirements_path.read_text(encoding="utf-8").splitlines()
+    existing_names = {_parse_requirement_name(line) for line in existing}
+    additions = [
+        dependency for dependency in dependencies
+        if _parse_requirement_name(dependency) not in existing_names
+    ]
+    if additions:
+        content = requirements_path.read_text(encoding="utf-8")
+        if content and not content.endswith("\n"):
+            content += "\n"
+        requirements_path.write_text(
+            content + "\n".join(additions) + "\n",
+            encoding="utf-8",
+        )
+
+
+def _add_engine_dependencies(
+    engine: str,
+    package_manager: str,
+    directory: Path,
+):
+    dependencies = ENGINE_DEPENDENCIES[engine]
+    if package_manager == "uv":
+        _run_uv(["add", *dependencies], directory)
+    else:
+        _add_pip_dependencies(dependencies, directory)
+
+
+def create_service(name, dataframe_engine: str | None = None):
+    if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*", name):
+        raise ValueError(
+            "service_name must start with a letter and contain only letters, "
+            "numbers, and underscores"
+        )
+
+    service_path = Path(name)
+    if service_path.exists():
+        response = input(
+            f"Service {name} already exists. Do you want to overwrite it? (y/N): "
+        ).strip().lower()
+        if response != "y":
+            print("Cancelled.")
+            return
+
+    dataframe_engine = dataframe_engine or _select_dataframe_engine()
+    if dataframe_engine not in SERVICE_TEMPLATES:
+        raise ValueError(f"Unsupported dataframe engine: {dataframe_engine}")
+
+    project_config = _load_project_config(Path.cwd())
+    _add_engine_dependencies(
+        dataframe_engine,
+        project_config["package_manager"],
+        Path.cwd(),
+    )
+
+    if service_path.exists():
+        shutil.rmtree(service_path)
+    service_path.mkdir()
+    (service_path / "__init__.py").touch()
+
+    class_name = to_pascal_case(name) + "Svc"
+    replacements = {
+        "#CLASS_NAME#": class_name,
+        "#MODULE_NAME#": name,
+    }
+    for src, destination_template in SERVICE_TEMPLATES[dataframe_engine].items():
+        destination_name = destination_template
+        for placeholder, value in replacements.items():
+            destination_name = destination_name.replace(placeholder, value)
+        _write_rendered_template(
+            src,
+            service_path / destination_name,
+            replacements,
+        )
+    _write_rendered_template(
+        "service/common/service.yaml",
+        service_path / "service.yaml",
+        replacements,
+    )
+
+    print(f"Service {name} created with {dataframe_engine}.")
+    if project_config["package_manager"] == "pip":
+        print("Install updated dependencies with `python -m pip install -r requirements.txt`.")
+    print(f"Run the generated service test for `{name}` before editing the template.")
+
+
 def _get_installed_pam_version() -> str:
     try:
         from pam import __version__
@@ -195,32 +350,5 @@ def _get_installed_pam_version() -> str:
         return "0.0.0"
 
 
-def _build_project_requirements() -> str:
-    pam_version = _get_installed_pam_version()
-    template = read_template_text("init/requirements.txt")
-    rendered = template.replace("#PAM_VERSION#", pam_version)
-    if not rendered.endswith("\n"):
-        rendered += "\n"
-    return rendered
-
-
-def _merge_requirements(existing_lines, frozen_lines):
-    """
-    Merge frozen requirements into existing lines while keeping existing entries.
-    """
-    existing_keys = {_parse_requirement_name(line) for line in existing_lines}
-    existing_keys.discard("")
-
-    merged = list(existing_lines)
-    for line in frozen_lines:
-        key = _parse_requirement_name(line)
-        if not key:
-            continue
-        if key not in existing_keys:
-            merged.append(line)
-            existing_keys.add(key)
-    return merged
-
-
 if __name__ == "__main__":
     main()

pam/models/request_command.py

@@ -26,6 +26,7 @@ import zipfile
 from flask import Request
 from pam.utils import log
 from pam.temp_file_utils import TempfileUtils
+from pam.request_file_format import RequestFileFormat
 
 @dataclass
 class RequestCommand:
@@ -43,6 +44,7 @@ class RequestCommand:
     next: str
     input_files: list[str]
     service_name: str
+    file_format: RequestFileFormat = RequestFileFormat.CSV
     log_api: str = ""
     sqlite_download_v2: str = ""
 
@@ -152,6 +154,7 @@ class RequestCommand:
 
         is_end = RequestCommand.__safe_str_to_bool(params.pop("is_end", "true"))
         next_page = params.pop("next", "")
+        file_format = RequestFileFormat.parse(params.pop("file_format", "csv"))
 
         input_files = []
         error_message = ""
@@ -179,6 +182,7 @@ class RequestCommand:
             next=next_page,
             input_files=input_files,
             service_name=service_name,
+            file_format=file_format,
             log_api=log_api,
             sqlite_download_v2=sqlite_download_v2
         ), error_message

pam/request_file_format.py

@@ -1,5 +1,14 @@
 from enum import Enum
 
 class RequestFileFormat(Enum):
-    CSV="csv"
-    PARQUET="parquet"
+    CSV = "csv"
+    PARQUET = "parquet"
+
+    @classmethod
+    def parse(cls, value):
+        if isinstance(value, cls):
+            return value
+        try:
+            return cls(str(value).strip().lower())
+        except ValueError:
+            return cls.CSV