deployml-core 0.0.56__tar.gz → 0.0.58__tar.gz

{deployml_core-0.0.56 → deployml_core-0.0.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deployml-core
-Version: 0.0.56
+Version: 0.0.58
 Summary: Infra for academia
 License-File: LICENSE
 Author: Drew Hoang

{deployml_core-0.0.56 → deployml_core-0.0.58}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "deployml-core"
-version = "0.0.56"
+version = "0.0.58"
 description = "Infra for academia"
 authors = [
     {name = "Drew Hoang", email = "codentell@gmail.com"},

{deployml_core-0.0.56 → deployml_core-0.0.58}/src/deployml/cli/cli.py

@@ -739,7 +739,6 @@ def generate():
         fg=typer.colors.BRIGHT_BLUE,
     )
 
-
 @cli.command()
 def terraform(
     action: str,

deployml_core-0.0.58/src/deployml/docker/fastapi/.dockerignore

@@ -0,0 +1,11 @@
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+env/
+venv/
+.venv
+.git
+.gitignore
+README.md

deployml_core-0.0.58/src/deployml/docker/fastapi/Dockerfile

@@ -0,0 +1,37 @@
+# Build for Linux/amd64 platform (not Apple Silicon)
+FROM --platform=linux/amd64 python:3.9-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+RUN pip install --no-cache-dir --upgrade pip setuptools wheel
+
+# Copy requirements and install
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY main.py .
+
+# Create non-root user
+RUN useradd -m -s /bin/bash fastapi && \
+    chown -R fastapi:fastapi /app
+
+# Switch to non-root user
+USER fastapi
+
+# Expose port
+EXPOSE 8000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the application
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

deployml_core-0.0.58/src/deployml/docker/fastapi/main.py

@@ -0,0 +1,127 @@
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from typing import List, Dict, Any, Optional
+import os
+from datetime import datetime
+import pandas as pd
+import mlflow
+import mlflow.pyfunc
+
+app = FastAPI(
+    title="FastAPI Demo",
+    description="Simple FastAPI application for Kubernetes deployment demo",
+    version="1.0.0"
+)
+
+# Hardcoded MLflow configuration
+MLFLOW_TRACKING_URI = "http://mlflow-service:5000"
+MODEL_NAME = "HousingPriceModel"
+
+# Global model variable
+model = None
+
+# Pydantic models
+class PredictionRequest(BaseModel):
+    features: Dict[str, float]
+
+class PredictionResponse(BaseModel):
+    prediction: float
+    timestamp: str
+    model_used: Optional[str] = None
+
+class HealthResponse(BaseModel):
+    status: str
+    timestamp: str
+    port: int
+    mlflow_connected: bool
+    model_loaded: bool
+
+def load_model_from_mlflow():
+    """Load model from MLflow Model Registry"""
+    global model
+    
+    try:
+        mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
+        model = mlflow.pyfunc.load_model(f"models:/{MODEL_NAME}/Production")
+        print(f"✓ Loaded model '{MODEL_NAME}' from Production stage")
+        return True
+    except Exception as e:
+        print(f"⚠️  Could not load model from MLflow: {e}")
+        model = None
+        return False
+
+# Load model at startup
+@app.on_event("startup")
+async def startup_event():
+    """Load MLflow model on startup"""
+    load_model_from_mlflow()
+
+@app.get("/")
+async def root():
+    """Root endpoint"""
+    return {
+        "service": "FastAPI Demo",
+        "version": "1.0.0",
+        "model_loaded": model is not None,
+        "model_name": MODEL_NAME,
+        "mlflow_uri": MLFLOW_TRACKING_URI,
+        "endpoints": {
+            "health": "/health",
+            "predict": "/predict",
+            "docs": "/docs"
+        }
+    }
+
+@app.get("/health", response_model=HealthResponse)
+async def health():
+    """Health check endpoint"""
+    port = int(os.getenv("FASTAPI_PORT", "8000"))
+    return HealthResponse(
+        status="healthy",
+        timestamp=datetime.now().isoformat(),
+        port=port,
+        mlflow_connected=True,
+        model_loaded=model is not None
+    )
+
+@app.post("/predict", response_model=PredictionResponse)
+async def predict(request: PredictionRequest):
+    """Prediction endpoint using MLflow model"""
+    global model
+    
+    # If model is not loaded, return -1
+    if model is None:
+        return PredictionResponse(
+            prediction=-1.0,
+            timestamp=datetime.now().isoformat(),
+            model_used=None
+        )
+    
+    try:
+        # Convert features dict to DataFrame in correct order
+        # Features: bedrooms, bathrooms, area_sqft, lot_size, year_built, city, state
+        feature_order = ['bedrooms', 'bathrooms', 'area_sqft', 'lot_size', 'year_built', 'city', 'state']
+        feature_values = [request.features.get(f, 0.0) for f in feature_order]
+        X = pd.DataFrame([feature_values], columns=feature_order)
+        
+        prediction = float(model.predict(X)[0])
+        model_used = f"MLflow: {MODEL_NAME}"
+        
+        return PredictionResponse(
+            prediction=prediction,
+            timestamp=datetime.now().isoformat(),
+            model_used=model_used
+        )
+    except Exception as e:
+        # On any error, return -1
+        return PredictionResponse(
+            prediction=-1.0,
+            timestamp=datetime.now().isoformat(),
+            model_used=None
+        )
+
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.getenv("FASTAPI_PORT", "8000"))
+    uvicorn.run(app, host="0.0.0.0", port=port)

deployml_core-0.0.58/src/deployml/docker/fastapi/requirements.txt

@@ -0,0 +1,7 @@
+fastapi==0.104.1
+uvicorn==0.24.0
+pydantic==2.5.2
+mlflow>=2.8.0
+scikit-learn>=1.0.0
+pandas>=1.3.0
+numpy>=1.21.0

deployml_core-0.0.58/src/deployml/docker/grafana-container/Dockerfile

@@ -0,0 +1,23 @@
+FROM grafana/grafana:10.4.5
+
+# Switch to root to copy files and change permissions
+USER root
+
+# Configure Grafana to listen on Cloud Run's expected port
+ENV GF_SERVER_HTTP_PORT=8080 \
+    GF_SECURITY_ADMIN_USER=admin \
+    GF_SECURITY_ADMIN_PASSWORD=admin
+
+# Copy entrypoint script and set permissions 
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod 755 /usr/local/bin/entrypoint.sh
+
+# Create provisioning directories
+RUN mkdir -p /etc/grafana/provisioning/datasources \
+    /etc/grafana/provisioning/dashboards
+
+# Switch back to the default grafana user for runtime
+USER grafana
+
+# Set custom entrypoint
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

deployml_core-0.0.58/src/deployml/docker/grafana-container/README.md

@@ -0,0 +1,3 @@
+docker build --platform=linux/amd64 -t gcr.io/mlops-intro-461805/grafana/grafana:latest .
+
+docker push gcr.io/mlops-intro-461805/grafana/grafana:latest

deployml_core-0.0.58/src/deployml/docker/grafana-container/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+set -e
+
+# Map Cloud Run's PORT → Grafana's expected var
+export GF_SERVER_HTTP_PORT="${PORT:-${GF_SERVER_HTTP_PORT:-8080}}"
+
+echo "Starting Grafana on port ${GF_SERVER_HTTP_PORT}..."
+[ -n "${GF_DATABASE_URL:-}" ] && echo "Using database: ${GF_DATABASE_URL}"
+
+exec /run.sh

deployml_core-0.0.58/src/deployml/docker/mlflow/Dockerfile

@@ -0,0 +1,39 @@
+# Build for Linux/amd64 platform (not Apple Silicon)
+FROM --platform=linux/amd64 python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+RUN pip install --no-cache-dir --upgrade pip setuptools wheel
+
+# Copy requirements and install
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Create non-root user
+RUN useradd -m -s /bin/bash mlflow && \
+    chown -R mlflow:mlflow /app
+
+# Create directories for MLflow data
+RUN mkdir -p /mlflow-artifacts && \
+    chown -R mlflow:mlflow /mlflow-artifacts
+
+# Switch to non-root user
+USER mlflow
+
+# Expose port
+EXPOSE 5000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
+    CMD curl -f http://localhost:5000/health || exit 1
+
+# Default command - MLflow server
+CMD ["mlflow", "server", "--host", "0.0.0.0", "--port", "5000"]
+

deployml_core-0.0.58/src/deployml/docker/mlflow/requirements.txt

@@ -0,0 +1,4 @@
+mlflow==3.10.0 
+psycopg2-binary 
+boto3 
+google-cloud-storage

deployml_core-0.0.58/src/deployml/notebook/docker.py

@@ -0,0 +1,224 @@
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+class ImageBuildError(Exception):
+    pass
+
+def build_images(
+    docker_root: Path,
+    gcp_project_id: Optional[str] = None,
+    region: str = "us-west1",
+    repository: str = "mlops-images",
+    tag: str = "latest",
+    create_repo: bool = False,
+    dry_run: bool = False,
+) -> None:
+    """
+    Build all Docker images located in subdirectories of docker_root.
+
+    Each subdirectory containing a Dockerfile will be built as a separate image.
+    The image name will match the subdirectory name.
+
+    Local mode:
+        Builds images locally using Docker.
+
+    GCP mode:
+        Uses Cloud Build and pushes images to Artifact Registry.
+
+    Args:
+        docker_root: Root folder containing subfolders with Dockerfiles.
+        gcp_project_id: If provided, use Cloud Build in this GCP project.
+        region: GCP region for Artifact Registry.
+        repository: Artifact Registry repository name.
+        tag: Docker image tag.
+        create_repo: Whether to create Artifact Registry repository (GCP mode only).
+        dry_run: If True, print commands without executing them.
+    """
+
+    docker_root = Path(docker_root)
+
+    if not docker_root.exists():
+        raise ValueError(f"Docker root does not exist: {docker_root}")
+
+    # Discover services
+    services = [
+        d for d in docker_root.iterdir()
+        if d.is_dir() and (d / "Dockerfile").exists()
+    ]
+
+    if not services:
+        print("No Dockerfiles found.")
+        return
+
+    print(f"Found {len(services)} service(s):")
+    for s in services:
+        print(f"  - {s.name}")
+
+    print()
+
+    # ----------------------------------------
+    # GCP MODE
+    # ----------------------------------------
+    if gcp_project_id:
+
+        image_base = f"{region}-docker.pkg.dev/{gcp_project_id}/{repository}"
+
+        # Create Artifact Registry repo if requested
+        if create_repo:
+            create_cmd = [
+                "gcloud", "artifacts", "repositories", "create", repository,
+                "--repository-format=docker",
+                "--location", region,
+                "--project", gcp_project_id,
+            ]
+
+            if dry_run:
+                print("Would create Artifact Registry repository:")
+                print("  " + " ".join(create_cmd))
+                print()
+            else:
+                print("Ensuring Artifact Registry repository exists...")
+                subprocess.run(create_cmd, check=False)  # safe if already exists
+                print()
+
+        # Build each service
+        for service_dir in services:
+            service_name = service_dir.name
+            image_uri = f"{image_base}/{service_name}:{tag}"
+
+            build_cmd = [
+                "gcloud", "builds", "submit",
+                str(service_dir),
+                "--tag", image_uri,
+                "--project", gcp_project_id,
+            ]
+
+            if dry_run:
+                print("Would submit Cloud Build:")
+                print("  " + " ".join(build_cmd))
+                print()
+            else:
+                print(f"Building {service_name} via Cloud Build...")
+                subprocess.run(build_cmd, check=True)
+                print(f"Pushed: {image_uri}")
+                print()
+
+    # ----------------------------------------
+    # LOCAL MODE
+    # ----------------------------------------
+    else:
+        for service_dir in services:
+            service_name = service_dir.name
+            image_name = f"{service_name}:{tag}"
+
+            build_cmd = [
+                "docker", "build",
+                "-t", image_name,
+                str(service_dir),
+            ]
+
+            if dry_run:
+                print("Would build locally:")
+                print("  " + " ".join(build_cmd))
+                print()
+            else:
+                print(f"Building {service_name} locally...")
+                subprocess.run(build_cmd, check=True)
+                print(f"Built: {image_name}")
+                print()
+
+    if dry_run:
+        print("Dry run complete. No commands were executed.")
+
+# -----------------------------
+# Local Docker Build
+# -----------------------------
+
+def _validate_docker():
+    try:
+        subprocess.run(
+            ["docker", "--version"],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except Exception:
+        raise ImageBuildError("Docker is not installed or not available in PATH.")
+
+
+def _build_locally(service_dirs: list[Path], tag: str):
+    print("Building images locally with Docker...\n")
+
+    for service_dir in service_dirs:
+        service_name = service_dir.name
+        image_name = f"{service_name}:{tag}"
+
+        print(f"Building {image_name} ...")
+
+        subprocess.run(
+            [
+                "docker",
+                "build",
+                "-t",
+                image_name,
+                str(service_dir),
+            ],
+            check=True,
+        )
+
+        print(f"Successfully built {image_name}\n")
+
+
+# -----------------------------
+# GCP Cloud Build
+# -----------------------------
+
+def _validate_gcloud():
+    try:
+        subprocess.run(
+            ["gcloud", "--version"],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except Exception:
+        raise ImageBuildError("gcloud CLI is not installed or not available in PATH.")
+
+
+def _build_with_cloud_build(
+    service_dirs: list[Path],
+    gcp_project_id: str,
+    region: str,
+    repository: str,
+    tag: str,
+):
+    print(f"Building images using Cloud Build in project '{gcp_project_id}'...\n")
+
+    for service_dir in service_dirs:
+        service_name = service_dir.name
+
+        image_uri = (
+            f"{region}-docker.pkg.dev/"
+            f"{gcp_project_id}/"
+            f"{repository}/"
+            f"{service_name}:{tag}"
+        )
+
+        print(f"Submitting Cloud Build for {image_uri} ...")
+
+        subprocess.run(
+            [
+                "gcloud",
+                "builds",
+                "submit",
+                str(service_dir),
+                "--tag",
+                image_uri,
+                "--project",
+                gcp_project_id,
+            ],
+            check=True,
+        )
+
+        print(f"Successfully built and pushed {image_uri}\n")

{deployml_core-0.0.56 → deployml_core-0.0.58}/src/deployml/templates/gcp/cloud_run/main.tf.j2

@@ -216,6 +216,20 @@ module "{{ stage_name }}_{{ tool.name }}" {
   {% endfor %}
 {% endfor %}
 
+# Bigquery table for features
+{% for stage in stack %}
+  {% for stage_name, tool in stage.items() %}
+    {% if stage_name == "data_store" and tool.name == "bigquery" %}
+    module "{{ stage_name }}_{{ tool.name }}" {
+  source     = "./modules/{{ tool.name }}/cloud/gcp"
+  project_id = var.project_id
+  region     = var.region
+  dataset_id = var.dataset_id
+}
+    {% endif %}
+  {% endfor %}
+{% endfor %}
+
 # Workflow orchestration (cron scheduling) module
 {% for stage in stack %}
   {% for stage_name, tool in stage.items() %}

deployml_core-0.0.58/src/deployml/terraform/modules/bigquery/cloud/gcp/main.tf

@@ -0,0 +1,38 @@
+resource "google_bigquery_dataset" "mlops" {
+  project    = var.project_id
+  dataset_id = var.dataset_id
+  location   = var.region
+}
+
+ resource "google_bigquery_table" "drift_metrics" {
+  dataset_id = google_bigquery_dataset.mlops.dataset_id
+  table_id   = "drift_metrics"
+  project    = var.project_id
+
+  schema = file("${path.module}/schemas/drift_metrics.json")
+}
+
+resource "google_bigquery_table" "ground_truth" {
+  dataset_id = google_bigquery_dataset.mlops.dataset_id
+  table_id   = "ground_truth"
+  project    = var.project_id
+
+  schema = file("${path.module}/schemas/ground_truth.json")
+}
+
+resource "google_bigquery_table" "offline_features" {
+  dataset_id = google_bigquery_dataset.mlops.dataset_id
+  table_id   = "offline_features"
+  project    = var.project_id
+
+  schema = file("${path.module}/schemas/offline_features.json")
+}
+
+resource "google_bigquery_table" "predictions" {
+  dataset_id = google_bigquery_dataset.mlops.dataset_id
+  table_id   = "predictions"
+  project    = var.project_id
+
+  schema = file("${path.module}/schemas/predictions.json")
+}
+

deployml_core-0.0.58/src/deployml/terraform/modules/bigquery/cloud/gcp/outputs.tf

@@ -0,0 +1,3 @@
+output "dataset_id" {
+  value = google_bigquery_dataset.mlops.dataset_id
+}

deployml_core-0.0.58/src/deployml/terraform/modules/bigquery/cloud/gcp/schemas/drift_metrics.json

@@ -0,0 +1,9 @@
+[
+  {"name": "metric_timestamp", "type": "TIMESTAMP", "mode": "REQUIRED"},
+  {"name": "metric_type", "type": "STRING", "mode": "REQUIRED"},
+  {"name": "feature_name", "type": "STRING", "mode": "NULLABLE"},
+  {"name": "value", "type": "FLOAT", "mode": "NULLABLE"},
+  {"name": "window_start", "type": "TIMESTAMP", "mode": "NULLABLE"},
+  {"name": "window_end", "type": "TIMESTAMP", "mode": "NULLABLE"},
+  {"name": "model_version", "type": "STRING", "mode": "NULLABLE"}
+]

deployml_core-0.0.58/src/deployml/terraform/modules/bigquery/cloud/gcp/schemas/ground_truth.json

@@ -0,0 +1,5 @@
+[
+  {"name": "entity_id", "type": "STRING", "mode": "REQUIRED"},
+  {"name": "event_timestamp", "type": "TIMESTAMP", "mode": "REQUIRED"},
+  {"name": "actual_value", "type": "FLOAT", "mode": "NULLABLE"}
+]

deployml_core-0.0.58/src/deployml/terraform/modules/bigquery/cloud/gcp/schemas/offline_features.json

@@ -0,0 +1,7 @@
+[
+  {"name": "entity_id", "type": "STRING", "mode": "REQUIRED"},
+  {"name": "event_timestamp", "type": "TIMESTAMP", "mode": "REQUIRED"},
+  {"name": "feature_1", "type": "FLOAT", "mode": "NULLABLE"},
+  {"name": "feature_2", "type": "FLOAT", "mode": "NULLABLE"},
+  {"name": "feature_3", "type": "FLOAT", "mode": "NULLABLE"}
+]

deployml_core-0.0.58/src/deployml/terraform/modules/bigquery/cloud/gcp/schemas/predictions.json

@@ -0,0 +1,8 @@
+[
+  {"name": "prediction_id", "type": "STRING", "mode": "REQUIRED"},
+  {"name": "entity_id", "type": "STRING", "mode": "REQUIRED"},
+  {"name": "prediction_timestamp", "type": "TIMESTAMP", "mode": "REQUIRED"},
+  {"name": "predicted_value", "type": "FLOAT", "mode": "NULLABLE"},
+  {"name": "model_version", "type": "STRING", "mode": "NULLABLE"},
+  {"name": "feature_snapshot_timestamp", "type": "TIMESTAMP", "mode": "NULLABLE"}
+]

deployml_core-0.0.58/src/deployml/terraform/modules/bigquery/cloud/gcp/variables.tf

@@ -0,0 +1,11 @@
+variable "project_id" {
+  type = string
+}
+
+variable "region" {
+  type = string
+}
+
+variable "dataset_id" {
+  type = string
+}

{deployml_core-0.0.56 → deployml_core-0.0.58}/src/deployml/utils/helpers.py

@@ -487,4 +487,4 @@ def _create_docker_folder(
 
     for item in templates.iterdir():
         if item.is_dir():
-            shutil.copytree(item, docker_target / item.name)
+            shutil.copytree(item, docker_target / item.name)