aissemble-inference-deploy 1.5.0rc3__py3-none-any.whl

aissemble_inference_deploy/generators/kubernetes.py

@@ -0,0 +1,119 @@
+###
+# #%L
+# aiSSEMBLE::Open Inference Protocol::Deploy
+# %%
+# Copyright (C) 2024 Booz Allen Hamilton Inc.
+# %%
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# #L%
+###
+"""
+Kubernetes deployment generator.
+
+Generates Kubernetes manifests with Kustomize structure for MLServer deployment.
+Includes base manifests and dev/prod overlays for environment-specific configuration.
+"""
+
+from pathlib import Path
+
+from .base import Generator, ModelInfo
+
+
+class KubernetesGenerator(Generator):
+    """Generator for Kubernetes deployment configurations using Kustomize."""
+
+    name = "kubernetes"
+
+    def generate(self, models: list[ModelInfo] | None = None) -> list[Path]:
+        """
+        Generate Kubernetes deployment configs with Kustomize structure.
+
+        Args:
+            models: Models to generate configs for (auto-detected if None)
+
+        Returns:
+            List of paths to generated files
+        """
+        if models is None:
+            models = self.detect_models()
+
+        generated_files = []
+        target_dir = self.output_dir / "kubernetes"
+
+        # Extract runtime packages for documentation
+        runtime_packages = self.extract_runtime_packages(models)
+
+        # Get image name (consistent with Docker generator)
+        image_name = self.get_image_name()
+
+        # Common template context
+        context = {
+            "models": models,
+            "runtime_packages": runtime_packages,
+            "image_name": image_name,
+            "app_name": image_name,
+            "http_port": 8080,
+            "grpc_port": 8081,
+            "node_port_http": 30080,
+            "node_port_grpc": 30081,
+        }
+
+        # Generate base manifests
+        base_dir = target_dir / "base"
+
+        deployment_content = self.render_template(
+            "kubernetes/deployment.yaml.j2", context
+        )
+        deployment_path = self.write_file(
+            base_dir / "deployment.yaml", deployment_content
+        )
+        generated_files.append(deployment_path)
+
+        service_content = self.render_template("kubernetes/service.yaml.j2", context)
+        service_path = self.write_file(base_dir / "service.yaml", service_content)
+        generated_files.append(service_path)
+
+        base_kustomization_content = self.render_template(
+            "kubernetes/kustomization.yaml.j2", context
+        )
+        base_kustomization_path = self.write_file(
+            base_dir / "kustomization.yaml", base_kustomization_content
+        )
+        generated_files.append(base_kustomization_path)
+
+        # Generate dev overlay
+        dev_dir = target_dir / "overlays" / "dev"
+        dev_kustomization_content = self.render_template(
+            "kubernetes/overlays/dev/kustomization.yaml.j2", context
+        )
+        dev_kustomization_path = self.write_file(
+            dev_dir / "kustomization.yaml", dev_kustomization_content
+        )
+        generated_files.append(dev_kustomization_path)
+
+        # Generate prod overlay
+        prod_dir = target_dir / "overlays" / "prod"
+        prod_kustomization_content = self.render_template(
+            "kubernetes/overlays/prod/kustomization.yaml.j2", context
+        )
+        prod_kustomization_path = self.write_file(
+            prod_dir / "kustomization.yaml", prod_kustomization_content
+        )
+        generated_files.append(prod_kustomization_path)
+
+        # Generate README
+        readme_content = self.render_template("kubernetes/README.md.j2", context)
+        readme_path = self.write_file(target_dir / "README.md", readme_content)
+        generated_files.append(readme_path)
+
+        return generated_files

aissemble_inference_deploy/generators/local.py

@@ -0,0 +1,162 @@
+###
+# #%L
+# aiSSEMBLE::Open Inference Protocol::Deploy
+# %%
+# Copyright (C) 2024 Booz Allen Hamilton Inc.
+# %%
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# #L%
+###
+"""
+Local MLServer deployment generator.
+"""
+
+import re
+from pathlib import Path
+
+from .base import Generator, ModelInfo
+
+
+class LocalGenerator(Generator):
+    """Generator for local MLServer deployment scripts."""
+
+    name = "local"
+
+    def generate(self, models: list[ModelInfo] | None = None) -> list[Path]:
+        """
+        Generate local MLServer run script.
+
+        Args:
+            models: Models to generate configs for (auto-detected if None)
+
+        Returns:
+            List of paths to generated files
+        """
+        if models is None:
+            models = self.detect_models()
+
+        generated_files = []
+        target_dir = self.output_dir / "local"
+
+        # From deploy/local, go up two levels (../.. = project_dir), then down to models
+        models_rel_path = "../../models"
+
+        # Validate path is safe (no shell metacharacters)
+        if not re.match(r"^[a-zA-Z0-9_/.-]+$", models_rel_path):
+            raise ValueError(f"Invalid models path: {models_rel_path}")
+
+        # Generate run script
+        script_content = self.render_template(
+            "local/run-mlserver.sh.j2",
+            {
+                "models_dir": models_rel_path,
+                "models": models,
+                "port": 8080,
+            },
+        )
+        script_path = self.write_file(
+            target_dir / "run-mlserver.sh",
+            script_content,
+            executable=True,
+        )
+        generated_files.append(script_path)
+
+        # Generate README
+        readme_content = self._generate_readme(models)
+        readme_path = self.write_file(target_dir / "README.md", readme_content)
+        generated_files.append(readme_path)
+
+        return generated_files
+
+    def _generate_readme(self, models: list[ModelInfo]) -> str:
+        """Generate README for local deployment."""
+        model_list = (
+            "\n".join(f"- {m.name}" for m in models) if models else "- (none detected)"
+        )
+
+        # Extract unique runtime packages for installation instructions
+        # Runtime is typically "module_name.ClassName", we need just the module
+        runtime_packages = set()
+        for model in models:
+            if model.runtime and "." in model.runtime:
+                # Extract package name from "aissemble_inference_sumy.SumyRuntime"
+                package = model.runtime.split(".")[0]
+                # Convert underscores to hyphens for PyPI package names
+                package = package.replace("_", "-")
+                runtime_packages.add(package)
+
+        runtime_install = ""
+        if runtime_packages:
+            packages_str = " ".join(sorted(runtime_packages))
+            runtime_install = f"""
+**Model-specific runtimes:**
+```bash
+pip install {packages_str}
+```
+"""
+
+        return f"""# Local MLServer Deployment
+
+This directory contains scripts for running your models locally with MLServer.
+
+## Prerequisites
+
+**Required:**
+- Python 3.11+
+- MLServer:
+  ```bash
+  pip install mlserver
+  ```
+{runtime_install}
+**Note:** The run script will check for MLServer and display installation instructions if not found.
+
+## Models
+
+{model_list}
+
+## Usage
+
+Start MLServer:
+
+```bash
+./run-mlserver.sh
+```
+
+The server will start on http://localhost:8080
+
+If MLServer is not installed, the script will display installation instructions.
+
+## Testing
+
+Once the server is running, you can test it with:
+
+```bash
+curl -X POST http://localhost:8080/v2/models/<model-name>/infer \\
+  -H "Content-Type: application/json" \\
+  -d '{{"inputs": [...]}}'
+```
+
+## Stopping
+
+Press Ctrl+C to stop the server.
+
+## Troubleshooting
+
+**MLServer not found:**
+- Ensure MLServer is installed: `pip install mlserver`
+- Check that it's in your PATH: `which mlserver`
+
+**Import errors when starting:**
+- Install model-specific dependencies (see Prerequisites above)
+- Verify your Python environment matches the model requirements (Python 3.11+)
+"""

aissemble_inference_deploy/registry.py

@@ -0,0 +1,158 @@
+###
+# #%L
+# aiSSEMBLE::Open Inference Protocol::Deploy
+# %%
+# Copyright (C) 2024 Booz Allen Hamilton Inc.
+# %%
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# #L%
+###
+"""
+Generator registry with entry point discovery.
+
+Allows custom generators to be registered via Python entry points,
+enabling extensibility for air-gapped systems, custom platforms, etc.
+"""
+
+import sys
+import threading
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .generators.base import Generator
+
+
+class GeneratorRegistry:
+    """
+    Registry for deployment generators discovered via entry points.
+
+    Generators register themselves via the 'inference.generators' entry point group:
+
+        [project.entry-points."inference.generators"]
+        openshift = "my_package:OpenShiftGenerator"
+
+    This allows custom generators to be installed as separate packages
+    without modifying the core aissemble-inference-deploy module.
+    """
+
+    _instance: "GeneratorRegistry | None" = None
+    _lock = threading.Lock()
+
+    def __init__(self):
+        self._generators: dict[str, type["Generator"]] | None = None
+
+    @classmethod
+    def instance(cls) -> "GeneratorRegistry":
+        """Get the singleton registry instance (thread-safe)."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:  # Double-check locking
+                    cls._instance = cls()
+        return cls._instance
+
+    @classmethod
+    def reset(cls) -> None:
+        """Reset the singleton instance (useful for testing)."""
+        with cls._lock:
+            cls._instance = None
+
+    def _discover_generators(self) -> dict[str, type["Generator"]]:
+        """Discover generators from entry points."""
+        # Import here to avoid circular imports
+        from .generators.base import Generator
+
+        generators: dict[str, type["Generator"]] = {}
+
+        try:
+            eps = entry_points(group="inference.generators")
+        except Exception as e:
+            # entry_points() failed - likely environment issue
+            print(
+                f"Warning: Failed to discover generators: {e}",
+                file=sys.stderr,
+            )
+            return generators
+
+        for ep in eps:
+            try:
+                generator_cls = ep.load()
+
+                # Validate it's actually a Generator subclass
+                if not isinstance(generator_cls, type):
+                    print(
+                        f"Warning: Generator '{ep.name}' is not a class: {type(generator_cls)}",
+                        file=sys.stderr,
+                    )
+                    continue
+
+                if not issubclass(generator_cls, Generator):
+                    print(
+                        f"Warning: Generator '{ep.name}' must be a subclass of Generator, got {generator_cls.__name__}",
+                        file=sys.stderr,
+                    )
+                    continue
+
+                generators[ep.name] = generator_cls
+
+            except ImportError as e:
+                print(
+                    f"Warning: Failed to import generator '{ep.name}': {e}",
+                    file=sys.stderr,
+                )
+            except Exception as e:
+                print(
+                    f"Warning: Failed to load generator '{ep.name}': {e}",
+                    file=sys.stderr,
+                )
+
+        return generators
+
+    @property
+    def generators(self) -> dict[str, type["Generator"]]:
+        """Get all available generators (lazy-loaded)."""
+        if self._generators is None:
+            self._generators = self._discover_generators()
+        return self._generators
+
+    def get(self, name: str) -> type["Generator"] | None:
+        """Get a generator by name, or None if not found."""
+        return self.generators.get(name)
+
+    def list_available(self) -> list[str]:
+        """List names of all available generators."""
+        return sorted(self.generators.keys())
+
+    def register(self, name: str, generator_cls: type["Generator"]) -> None:
+        """
+        Manually register a generator (useful for testing or runtime registration).
+
+        Args:
+            name: The target name (e.g., 'docker', 'kubernetes')
+            generator_cls: The generator class
+
+        Raises:
+            TypeError: If generator_cls is not a Generator subclass
+        """
+        from .generators.base import Generator
+
+        if not isinstance(generator_cls, type) or not issubclass(
+            generator_cls, Generator
+        ):
+            raise TypeError(
+                f"generator_cls must be a subclass of Generator, got {type(generator_cls)}"
+            )
+
+        if self._generators is None:
+            self._generators = self._discover_generators()
+        self._generators[name] = generator_cls

aissemble_inference_deploy/templates/docker/.dockerignore.j2

@@ -0,0 +1,47 @@
+# Generated by aissemble-inference-deploy
+# Files to exclude from Docker build context
+
+# Version control
+.git
+.gitignore
+
+# Python
+__pycache__
+*.py[cod]
+*$py.class
+*.so
+.Python
+.venv
+venv/
+ENV/
+env/
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Build artifacts
+*.egg-info/
+dist/
+build/
+target/
+
+# Test artifacts
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Documentation
+docs/_build/
+*.md
+!README.md
+
+# Deployment configs (we only need models/)
+deploy/
+
+# OS files
+.DS_Store
+Thumbs.db

aissemble_inference_deploy/templates/docker/Dockerfile.j2

@@ -0,0 +1,59 @@
+# Generated by aissemble-inference-deploy
+# Multi-stage Dockerfile for MLServer deployment
+{% if use_wheels %}
+# Dev mode: Dependencies built with uv-monorepo-dependency-tool
+{% else %}
+# Release mode: Dependencies installed from PyPI
+{% endif %}
+
+# Build stage - install dependencies using uv
+FROM python:{{ python_version }}-slim AS builder
+
+# Install uv
+RUN pip install --no-cache-dir uv
+
+WORKDIR /app
+
+{% if use_wheels %}
+# Copy pre-built wheels with pinned dependencies (dev mode)
+COPY deploy/docker/wheels/ /wheels/
+
+# Create virtual environment and install wheels
+RUN uv venv .venv && \
+    uv pip install --no-cache /wheels/*.whl
+{% else %}
+# Copy requirements file (release mode)
+COPY deploy/docker/requirements.txt .
+
+# Create virtual environment and install from PyPI
+RUN uv venv .venv && \
+    uv pip install --no-cache -r requirements.txt
+{% endif %}
+
+# Runtime stage - minimal image
+FROM python:{{ python_version }}-slim
+
+WORKDIR /app
+
+# Copy virtual environment from builder
+COPY --from=builder /app/.venv /app/.venv
+
+# Add venv to PATH
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Copy models directory
+COPY models/ /app/models/
+
+# Create MLServer settings to bind to all interfaces (required for Docker)
+# parallel_workers=0 disables multiprocessing to avoid uvloop issues
+RUN echo '{"host": "0.0.0.0", "http_port": 8080, "grpc_port": 8081, "parallel_workers": 0}' > /app/models/settings.json
+
+# Expose MLServer ports
+EXPOSE 8080 8081
+
+# Health check using MLServer's ready endpoint
+HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/v2/health/ready')" || exit 1
+
+# Start MLServer
+CMD ["mlserver", "start", "/app/models"]

aissemble_inference_deploy/templates/docker/README.md.j2

@@ -0,0 +1,163 @@
+# Docker Deployment
+
+This directory contains Docker configuration for containerized deployment.
+
+{% if use_wheels %}
+**Mode:** Dev (local wheels)
+{% else %}
+**Mode:** Release (PyPI packages)
+{% endif %}
+
+## Prerequisites
+
+- Docker Desktop or Rancher Desktop
+- Docker Compose (included with Docker Desktop)
+
+## Models
+
+{% for model in models %}
+- {{ model.name }}
+{% else %}
+- (none detected)
+{% endfor %}
+
+## Runtime Dependencies
+
+{% if use_wheels %}
+The following packages are installed in the container (from pre-built wheels):
+
+{% for package in runtime_packages %}
+- {{ package }}
+{% endfor %}
+
+The wheels in `wheels/` were built using [uv-monorepo-dependency-tool](https://github.com/TechnologyBrewery/uv-monorepo-dependency-tool)
+to convert local path dependencies to pinned version dependencies.
+{% else %}
+The following packages are installed from PyPI:
+
+{% for package in runtime_packages %}
+- {{ package }}
+{% endfor %}
+{% endif %}
+
+## Quick Start
+
+Build and start the container:
+
+```bash
+docker-compose up --build
+```
+
+The server will be available at http://localhost:{{ http_port }}
+
+## Usage
+
+### Build Only
+
+```bash
+docker-compose build
+```
+
+### Start in Background
+
+```bash
+docker-compose up -d
+```
+
+### View Logs
+
+```bash
+docker-compose logs -f
+```
+
+### Stop
+
+```bash
+docker-compose down
+```
+
+## Testing
+
+Once the container is running, test the endpoint:
+
+```bash
+curl -X POST http://localhost:{{ http_port }}/v2/models/<model-name>/infer \
+  -H "Content-Type: application/json" \
+  -d '{"inputs": [...]}'
+```
+
+## Customization
+
+### Environment Variables
+
+The docker-compose.yml supports these environment variables:
+
+- `MLSERVER_HTTP_PORT`: HTTP port (default: {{ http_port }})
+- `MLSERVER_GRPC_PORT`: gRPC port (default: {{ grpc_port }})
+
+### Resource Limits
+
+Edit docker-compose.yml to add resource constraints:
+
+```yaml
+services:
+  mlserver:
+    deploy:
+      resources:
+        limits:
+          cpus: '2'
+          memory: 4G
+```
+
+{% if use_wheels %}
+### Rebuilding Wheels
+
+To regenerate the wheels with updated dependencies:
+
+```bash
+uv tool run uv-monorepo-dependency-tool build-rewrite-path-deps --version-pinning-strategy=mixed
+cp dist/*.whl deploy/docker/wheels/
+```
+
+Or re-run the generator:
+
+```bash
+inference deploy init --target docker
+```
+{% else %}
+### Updating Dependencies
+
+Edit `requirements.txt` to change package versions, then rebuild:
+
+```bash
+docker-compose build --no-cache
+```
+{% endif %}
+
+## Production Considerations
+
+For production deployments:
+
+1. **Multi-architecture builds**: Use `docker buildx` for ARM/AMD64 support
+2. **Registry**: Push to a container registry (Docker Hub, ECR, GCR, etc.)
+3. **Secrets**: Use Docker secrets or environment variable injection
+4. **Health checks**: The Dockerfile includes MLServer health endpoints
+5. **Logging**: Configure structured logging for your platform
+
+## Troubleshooting
+
+**Build fails with package not found:**
+{% if use_wheels %}
+- Regenerate wheels: `inference deploy init --target docker`
+{% else %}
+- Ensure runtime packages are published to PyPI
+- Check network connectivity to PyPI
+{% endif %}
+
+**Container exits immediately:**
+- Check logs: `docker-compose logs`
+- Verify model-settings.json files are valid JSON
+
+**Port already in use:**
+- Stop other services on port {{ http_port }}/{{ grpc_port }}
+- Or change the ports in docker-compose.yml