deployml-core 0.0.58__tar.gz → 0.0.59__tar.gz

{deployml_core-0.0.58 → deployml_core-0.0.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deployml-core
-Version: 0.0.58
+Version: 0.0.59
 Summary: Infra for academia
 License-File: LICENSE
 Author: Drew Hoang

{deployml_core-0.0.58 → deployml_core-0.0.59}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "deployml-core"
-version = "0.0.58"
+version = "0.0.59"
 description = "Infra for academia"
 authors = [
     {name = "Drew Hoang", email = "codentell@gmail.com"},

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/cli/cli.py

@@ -788,6 +788,9 @@ def deploy(
     generate_only: bool = typer.Option(
         False, "--generate-only", "-g", help="Only generate manifests, do not apply (for GKE deployments)"
     ),
+    verbose: bool = typer.Option(
+        False, "--verbose", "-v", help="Stream Terraform logs to stdout instead of showing progress bar"
+    ),
 ):
     """
     Deploy infrastructure based on a YAML configuration file.
@@ -1027,6 +1030,34 @@ def deploy(
 
     typer.echo(f" Unified bucket creation: {create_artifact_bucket}")
 
+    # Auto-resolve image URIs: if a service image is not specified in the config (or still
+    # points to a legacy gcr.io path), derive it from the Artifact Registry path that
+    # `deployml build-images` produces: {region}-docker.pkg.dev/{project}/mlops-images/{name}
+    # This keeps the YAML config free of hardcoded image URIs — project/region are enough.
+    _TOOL_IMAGE_NAMES = {
+        "mlflow": "mlflow",
+        "feast": "feast",
+        "fastapi": "fastapi",
+        "grafana": "grafana-container",
+        "wandb": "wandb",
+    }
+    _ar_base = f"{region}-docker.pkg.dev/{project_id}/mlops-images"
+    for stage in stack:
+        for stage_name, tool in stage.items():
+            tool_name = tool.get("name", "")
+            params = tool.setdefault("params", {})
+            existing_image = params.get("image", "")
+            if not existing_image or existing_image.startswith("gcr.io/"):
+                image_name = _TOOL_IMAGE_NAMES.get(tool_name)
+                if image_name:
+                    params["image"] = f"{_ar_base}/{image_name}:latest"
+            # Cron job images are per-job and must be set explicitly — skip here
+            if stage_name == "workflow_orchestration" and tool_name == "cron":
+                for job in params.get("jobs", []):
+                    if not job.get("image") or job.get("image", "").startswith("gcr.io/"):
+                        job_name = job.get("service_name", "")
+                        job["image"] = f"{_ar_base}/{job_name}:latest"
+
     env = Environment(loader=FileSystemLoader(TEMPLATE_DIR))
     # PATCH: Use wandb_main.tf.j2 or mlflow_main.tf.j2 for cloud_run if present
     if deployment_type == "cloud_run":
@@ -1253,6 +1284,7 @@ def deploy(
             ["terraform", "apply", "-auto-approve"],
             DEPLOYML_TERRAFORM_DIR,
             minutes,
+            verbose=verbose,
         )
         if result_code == 0:
             typer.echo(" Deployment complete!")
@@ -1421,6 +1453,84 @@ def deploy(
         typer.echo(" Deployment cancelled")
 
 
+@cli.command()
+def get_urls(
+    config_path: Path = typer.Option(
+        ..., "--config-path", "-c", help="Path to YAML config file"
+    ),
+    env_path: Path = typer.Option(
+        Path(".env"), "--env-path", help="Path to write .env file"
+    ),
+):
+    """
+    Print service URLs from the last deployment and write them to a .env file.
+    """
+    if not config_path.exists():
+        typer.echo(f" Config file not found: {config_path}")
+        raise typer.Exit(code=1)
+
+    config = yaml.safe_load(config_path.read_text())
+    workspace_name = config.get("name") or "development"
+    terraform_dir = Path.cwd() / ".deployml" / workspace_name / "terraform"
+
+    if not terraform_dir.exists():
+        typer.echo(f" No deployment found at {terraform_dir}. Run 'deployml deploy' first.")
+        raise typer.Exit(code=1)
+
+    output_proc = subprocess.run(
+        ["terraform", "output", "-json"],
+        cwd=terraform_dir,
+        capture_output=True,
+        text=True,
+    )
+    if output_proc.returncode != 0:
+        typer.echo(" Could not retrieve Terraform outputs. Is the stack deployed?")
+        raise typer.Exit(code=1)
+
+    try:
+        outputs = json.loads(output_proc.stdout)
+    except Exception:
+        typer.echo(" Failed to parse Terraform outputs.")
+        raise typer.Exit(code=1)
+
+    if not outputs:
+        typer.echo("No outputs found. Is the stack deployed?")
+        raise typer.Exit(code=1)
+
+    typer.echo("\n DeployML Outputs:")
+    env_lines = []
+    for key, value in outputs.items():
+        is_sensitive = value.get("sensitive", False)
+        output_val = value.get("value")
+        env_key = key.upper()
+        if is_sensitive:
+            typer.secho(f"  {key}: [SENSITIVE] (value hidden)", fg=typer.colors.YELLOW)
+        elif isinstance(output_val, str):
+            if output_val.startswith("http://") or output_val.startswith("https://"):
+                typer.secho(f"  {key}: {output_val}", fg=typer.colors.BRIGHT_BLUE, bold=True)
+                env_lines.append(f"{env_key}={output_val}")
+            elif output_val == "":
+                typer.secho(f"  {key}: [No value]", fg=typer.colors.YELLOW)
+            else:
+                typer.echo(f"  {key}: {output_val}")
+                env_lines.append(f"{env_key}={output_val}")
+        elif isinstance(output_val, dict):
+            typer.echo(f"  {key}:")
+            for subkey, subval in output_val.items():
+                sub_env_key = f"{env_key}_{subkey.upper()}"
+                if isinstance(subval, str) and (subval.startswith("http://") or subval.startswith("https://")):
+                    typer.secho(f"    {subkey}: {subval}", fg=typer.colors.BRIGHT_BLUE, bold=True)
+                    env_lines.append(f"{sub_env_key}={subval}")
+                elif isinstance(subval, str) and subval:
+                    typer.echo(f"    {subkey}: {subval}")
+                    env_lines.append(f"{sub_env_key}={subval}")
+        else:
+            typer.echo(f"  {key}: {output_val}")
+
+    env_path.write_text("\n".join(env_lines) + "\n")
+    typer.echo(f"\n .env written to {env_path.resolve()}")
+
+
 @cli.command()
 def destroy(
     config_path: Path = typer.Option(
@@ -1487,16 +1597,53 @@ def destroy(
             cwd=DEPLOYML_TERRAFORM_DIR,
         )
 
-        # Check if we have Cloud SQL resources and clean them up first
-        plan_result = subprocess.run(
-            ["terraform", "plan", "-destroy"],
+        # Shut down Cloud Run services first to close any open DB connections
+        # before attempting to destroy Cloud SQL — otherwise active connections
+        # prevent database/user deletion and the destroy fails.
+        region = config.get("provider", {}).get("region", "us-central1")
+        cr_result = subprocess.run(
+            ["gcloud", "run", "services", "list",
+             "--project", project_id,
+             "--region", region,
+             "--format", "value(metadata.name)"],
+            capture_output=True, text=True
+        )
+        if cr_result.returncode == 0:
+            services = [s.strip() for s in cr_result.stdout.splitlines() if s.strip()]
+            for service in services:
+                typer.echo(f" Deleting Cloud Run service: {service}")
+                subprocess.run(
+                    ["gcloud", "run", "services", "delete", service,
+                     "--project", project_id,
+                     "--region", region,
+                     "--quiet"],
+                    capture_output=True,
+                )
+
+        # Remove Cloud SQL databases and user from Terraform state so Terraform
+        # doesn't try to delete them individually — the instance deletion handles
+        # that automatically, avoiding active-connection errors on destroy.
+        state_result = subprocess.run(
+            ["terraform", "state", "list"],
             cwd=DEPLOYML_TERRAFORM_DIR,
             capture_output=True,
             text=True,
         )
-
-        if "google_sql_database_instance" in plan_result.stdout:
-            cleanup_cloud_sql_resources(DEPLOYML_TERRAFORM_DIR, project_id)
+        if state_result.returncode == 0:
+            resources_to_remove = [
+                r.strip() for r in state_result.stdout.splitlines()
+                if any(x in r for x in [
+                    "google_sql_database.",
+                    "google_sql_user.",
+                ])
+            ]
+            for resource in resources_to_remove:
+                typer.echo(f" Removing from state: {resource}")
+                subprocess.run(
+                    ["terraform", "state", "rm", resource],
+                    cwd=DEPLOYML_TERRAFORM_DIR,
+                    capture_output=True,
+                )
 
         # Build destroy command
         cmd = ["terraform", "destroy", "--auto-approve"]

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/docker/fastapi/Dockerfile

@@ -1,5 +1,5 @@
 # Build for Linux/amd64 platform (not Apple Silicon)
-FROM --platform=linux/amd64 python:3.9-slim
+FROM python:3.9-slim
 
 # Set working directory
 WORKDIR /app
@@ -34,4 +34,5 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
     CMD curl -f http://localhost:8000/health || exit 1
 
 # Run the application
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+# Uses PORT env var so it works on Cloud Run (PORT=8080) and locally (PORT=8000)
+CMD uvicorn main:app --host 0.0.0.0 --port ${PORT:-8000}

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/docker/fastapi/main.py

@@ -14,9 +14,9 @@ app = FastAPI(
     version="1.0.0"
 )
 
-# Hardcoded MLflow configuration
-MLFLOW_TRACKING_URI = "http://mlflow-service:5000"
-MODEL_NAME = "HousingPriceModel"
+# MLflow configuration — override via MLFLOW_TRACKING_URI env var
+MLFLOW_TRACKING_URI = os.getenv("MLFLOW_TRACKING_URI", "http://mlflow-service:5000")
+MODEL_NAME = os.getenv("MODEL_NAME", "HousingPriceModel")
 
 # Global model variable
 model = None

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/docker/mlflow/Dockerfile

@@ -1,5 +1,5 @@
 # Build for Linux/amd64 platform (not Apple Silicon)
-FROM --platform=linux/amd64 python:3.11-slim
+FROM python:3.11-slim
 
 # Set working directory
 WORKDIR /app
@@ -35,5 +35,6 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
     CMD curl -f http://localhost:5000/health || exit 1
 
 # Default command - MLflow server
-CMD ["mlflow", "server", "--host", "0.0.0.0", "--port", "5000"]
+# Uses PORT env var so it works on Cloud Run (PORT=8080) and locally (PORT=5000)
+CMD mlflow server --host 0.0.0.0 --port ${PORT:-5000} --dev
 

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/templates/gcp/cloud_run/mlflow_main.tf.j2

@@ -1,6 +1,6 @@
 {% extends "gcp/cloud_run/base_main.tf.j2" %}
 
-{% set flags = namespace(needs_postgres=false) %}
+{% set flags = namespace(needs_postgres=false, has_grafana=false) %}
 {% for stage in stack %}
   {% for stage_name, tool in stage.items() %}
     {% if tool.name == "mlflow" and tool.params.get("backend_store_uri", "") == "postgresql" %}
@@ -9,6 +9,9 @@
     {% if tool.name == "feast" and tool.params.get("backend_store_uri", "") == "postgresql" %}
       {% set flags.needs_postgres = true %}
     {% endif %}
+    {% if stage_name == "model_monitoring" and tool.name == "grafana" %}
+      {% set flags.has_grafana = true %}
+    {% endif %}
   {% endfor %}
 {% endfor %}
 
@@ -44,6 +47,8 @@ module "cloud_sql_postgres" {
   db_instance_name = "mlflow-postgres-{{ project_id }}"
   db_name         = "mlflow"
   db_user         = "mlflow"
+  # Create the metrics DB when Grafana is in the stack so it has a backend to connect to
+  create_metrics_db = {{ "true" if flags.has_grafana else "false" }}
 }
 
 
@@ -288,69 +293,57 @@ module "{{ stage_name }}_{{ tool.name }}" {
     {% endif %}
   {% endfor %}
 {% endfor %}
+module "bigquery" {
+  source     = "./modules/bigquery/cloud/gcp"
+  project_id = var.project_id
+  region     = var.region
+  dataset_id = "mlops"
+}
 {% endblock %}
 
 {% block outputs %}
+# MLflow — single URL and bucket regardless of how many stack stages use it
+{% set mlflow_stage = namespace(found=false, name="") %}
 {% for stage in stack %}
   {% for stage_name, tool in stage.items() %}
-    {% if tool.name == "mlflow" %}
-output "{{ stage_name }}_mlflow_url" {
-  value = var.enable_{{ stage_name }}_mlflow && length(module.{{ stage_name }}_mlflow) > 0 ? module.{{ stage_name }}_mlflow[0].service_url : ""
-}
-output "{{ stage_name }}_mlflow_bucket" {
-  value = var.enable_{{ stage_name }}_mlflow && length(module.{{ stage_name }}_mlflow) > 0 ? module.{{ stage_name }}_mlflow[0].bucket_name : ""
-}
+    {% if tool.name == "mlflow" and not mlflow_stage.found %}
+      {% set mlflow_stage.found = true %}
+      {% set mlflow_stage.name = stage_name %}
     {% endif %}
   {% endfor %}
 {% endfor %}
-{% if create_artifact_bucket %}
-{% for stage in stack %}
-  {% for stage_name, tool in stage.items() %}
-    {% if tool.name == "mlflow" and tool.params.get("artifact_bucket") %}
-output "{{ stage_name }}_mlflow_artifact_bucket" {
-  value = google_storage_bucket.{{ stage_name }}_mlflow_artifact.name
+{% if mlflow_stage.found %}
+output "mlflow_url" {
+  value = var.enable_{{ mlflow_stage.name }}_mlflow && length(module.{{ mlflow_stage.name }}_mlflow) > 0 ? module.{{ mlflow_stage.name }}_mlflow[0].service_url : ""
+}
+output "artifact_bucket" {
+  value = var.enable_{{ mlflow_stage.name }}_mlflow && length(module.{{ mlflow_stage.name }}_mlflow) > 0 ? module.{{ mlflow_stage.name }}_mlflow[0].bucket_name : ""
 }
-    {% endif %}
-  {% endfor %}
-{% endfor %}
 {% endif %}
 {% if flags.needs_postgres %}
-output "instance_connection_name" {
-  value = module.cloud_sql_postgres.instance_connection_name
-}
 output "postgresql_credentials" {
   value = module.cloud_sql_postgres.postgresql_credentials
   sensitive = true
 }
 {% endif %}
+output "bigquery_dataset" {
+  value = module.bigquery.dataset_id
+}
 
-# Output for Grafana if present
 {% for stage in stack %}
   {% for stage_name, tool in stage.items() %}
-    {% if stage_name == "model_monitoring" and tool.name == "grafana" %}
-output "{{ stage_name }}_{{ tool.name }}_url" {
+    {% if stage_name == "model_serving" and tool.name == "fastapi" %}
+output "fastapi_url" {
   value = length(module.{{ stage_name }}_{{ tool.name }}) > 0 ? module.{{ stage_name }}_{{ tool.name }}[0].service_url : ""
 }
     {% endif %}
-  {% endfor %}
-{% endfor %}
-
-# Output for FastAPI model serving if present
-{% for stage in stack %}
-  {% for stage_name, tool in stage.items() %}
-    {% if stage_name == "model_serving" and tool.name == "fastapi" %}
-output "{{ stage_name }}_{{ tool.name }}_url" {
+    {% if stage_name == "model_monitoring" and tool.name == "grafana" %}
+output "grafana_url" {
   value = length(module.{{ stage_name }}_{{ tool.name }}) > 0 ? module.{{ stage_name }}_{{ tool.name }}[0].service_url : ""
 }
     {% endif %}
-  {% endfor %}
-{% endfor %}
-
-# Output for Feast feature store if present
-{% for stage in stack %}
-  {% for stage_name, tool in stage.items() %}
     {% if stage_name == "feature_store" and tool.name == "feast" %}
-output "{{ stage_name }}_{{ tool.name }}_url" {
+output "feast_url" {
   value = length(module.{{ stage_name }}_{{ tool.name }}) > 0 ? module.{{ stage_name }}_{{ tool.name }}[0].service_url : ""
 }
     {% endif %}

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/terraform/modules/bigquery/cloud/gcp/main.tf

@@ -10,6 +10,7 @@ resource "google_bigquery_dataset" "mlops" {
   project    = var.project_id
 
   schema = file("${path.module}/schemas/drift_metrics.json")
+  deletion_protection = false
 }
 
 resource "google_bigquery_table" "ground_truth" {
@@ -18,6 +19,7 @@ resource "google_bigquery_table" "ground_truth" {
   project    = var.project_id
 
   schema = file("${path.module}/schemas/ground_truth.json")
+  deletion_protection = false
 }
 
 resource "google_bigquery_table" "offline_features" {
@@ -26,6 +28,7 @@ resource "google_bigquery_table" "offline_features" {
   project    = var.project_id
 
   schema = file("${path.module}/schemas/offline_features.json")
+  deletion_protection = false
 }
 
 resource "google_bigquery_table" "predictions" {
@@ -34,5 +37,6 @@ resource "google_bigquery_table" "predictions" {
   project    = var.project_id
 
   schema = file("${path.module}/schemas/predictions.json")
+  deletion_protection = false
 }
 

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/terraform/modules/cloud_sql_postgres/main.tf

@@ -90,7 +90,7 @@ resource "google_sql_database" "feast_db" {
   instance = google_sql_database_instance.postgres.name
   project  = var.project_id
   depends_on = [null_resource.verify_instance_running]
-  
+
   lifecycle {
     ignore_changes = [name]
   }
@@ -102,7 +102,7 @@ resource "google_sql_database" "metrics_db" {
   instance = google_sql_database_instance.postgres.name
   project  = var.project_id
   depends_on = [null_resource.verify_instance_running]
-  
+
   lifecycle {
     ignore_changes = [name]
   }
@@ -114,9 +114,8 @@ resource "google_sql_user" "users" {
   password = random_password.db_password.result
   project  = var.project_id
   depends_on = [null_resource.verify_instance_running]
-  
+
   lifecycle {
-    # Retry on failure in case instance is still starting
     create_before_destroy = true
   }
 }

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/terraform/modules/mlflow/cloud/gcp/cloud_run/main.tf

@@ -38,11 +38,18 @@ resource "google_cloud_run_service" "mlflow" {
           name  = "MLFLOW_SERVER_HOST"
           value = "0.0.0.0"
         }
-        
+
         env {
           name  = "MLFLOW_SERVER_PORT"
           value = "8080"
         }
+
+        # Allow all host headers — required for Cloud Run since the Host header
+        # is the dynamic *.run.app URL which MLflow's DNS rebinding check rejects by default
+        env {
+          name  = "MLFLOW_SERVER_ALLOWED_HOSTS"
+          value = "*"
+        }
         
         # Backend store URI
         dynamic "env" {

{deployml_core-0.0.58 → deployml_core-0.0.59}/src/deployml/utils/helpers.py

@@ -99,9 +99,19 @@ def copy_modules_to_workspace(
     if teardown_enabled:
         used_modules.add("teardown")
 
+    # BigQuery is always included — provides the mlops dataset and tables
+    used_modules.add("bigquery")
+
     # Only copy the modules that are being used, and only the specific deployment type
     for module_path in MODULE_TEMPLATES_DIR.iterdir():
         if module_path.is_dir() and module_path.name in used_modules:
+            # Special case: always copy full bigquery module
+            if module_path.name == "bigquery":
+                dest_module_path = modules_dir / module_path.name
+                if dest_module_path.exists():
+                    shutil.rmtree(dest_module_path)
+                shutil.copytree(module_path, dest_module_path)
+                continue
             # Special case: always copy full cloud_sql_postgres module
             if module_path.name == "cloud_sql_postgres":
                 dest_module_path = modules_dir / module_path.name
@@ -257,97 +267,38 @@ def estimate_terraform_time(plan_output: str, operation: str = "apply") -> str:
 
 def cleanup_cloud_sql_resources(terraform_dir: Path, project_id: str):
     """
-    Clean up Cloud SQL database and user before destroying the instance.
+    Terminate all Cloud SQL connections before destroy so Terraform can cleanly
+    delete databases and users. We just restart the instance — that kills all
+    active connections — and let Terraform handle the actual resource deletion.
     """
     import subprocess
     import time as _time
 
     try:
-        # Get the instance name from terraform state
         result = subprocess.run(
             ["terraform", "output", "-raw", "instance_connection_name"],
             cwd=terraform_dir,
             capture_output=True,
             text=True,
         )
-        if result.returncode == 0:
-            instance_connection_name = result.stdout.strip()
-            # Extract instance name from connection name (format: project:region:instance)
-            parts = instance_connection_name.split(":")
-            instance_name = parts[2] if len(parts) == 3 else instance_connection_name
-
-            print("🗄️  Cleaning up Cloud SQL resources (terminate connections, drop DBs, drop users)...")
-
-            # Restart instance to terminate all connections (most reliable non-interactive method)
-            restart_cmd = [
-                "gcloud",
-                "sql",
-                "instances",
-                "restart",
-                instance_name,
-                "--project",
-                project_id,
-                "--quiet",
-            ]
-            subprocess.run(restart_cmd, capture_output=True, text=True)
-            _time.sleep(5)
-
-            # List existing databases to avoid failing on non-existent ones
-            list_db_cmd = [
-                "gcloud",
-                "sql",
-                "databases",
-                "list",
-                "--instance",
-                instance_name,
-                "--project",
-                project_id,
-                "--format=value(name)",
-            ]
-            list_proc = subprocess.run(list_db_cmd, capture_output=True, text=True)
-            existing_dbs = set(list_proc.stdout.strip().splitlines()) if list_proc.returncode == 0 else set()
-
-            # Attempt to delete known course DBs if present
-            target_dbs = ["mlflow", "feast", "metrics"]
-            for db_name in target_dbs:
-                if db_name in existing_dbs:
-                    delete_db_cmd = [
-                        "gcloud",
-                        "sql",
-                        "databases",
-                        "delete",
-                        db_name,
-                        "--instance",
-                        instance_name,
-                        "--project",
-                        project_id,
-                        "--quiet",
-                    ]
-                    # Retry a few times in case connections are still draining
-                    for attempt in range(3):
-                        proc = subprocess.run(delete_db_cmd, capture_output=True, text=True)
-                        if proc.returncode == 0:
-                            break
-                        _time.sleep(5)
-
-            # Drop common users after DBs are removed
-            target_users = ["mlflow", "feast", "metrics"]
-            for user in target_users:
-                drop_user_cmd = [
-                    "gcloud",
-                    "sql",
-                    "users",
-                    "delete",
-                    user,
-                    "--instance",
-                    instance_name,
-                    "--project",
-                    project_id,
-                    "--quiet",
-                ]
-                subprocess.run(drop_user_cmd, capture_output=True, text=True)
-
-            print("✅ Cloud SQL cleanup completed (best-effort)")
+        if result.returncode != 0:
+            return
+
+        instance_connection_name = result.stdout.strip()
+        parts = instance_connection_name.split(":")
+        instance_name = parts[2] if len(parts) == 3 else instance_connection_name
+
+        print(f"🗄️  Restarting Cloud SQL instance to close active connections: {instance_name}")
+        subprocess.run(
+            ["gcloud", "sql", "instances", "restart", instance_name,
+             "--project", project_id, "--quiet"],
+            capture_output=True,
+            text=True,
+        )
+        # Give the instance time to fully restart and drop all connections
+        _time.sleep(30)
+        print("✅ Cloud SQL connections cleared")
+
     except Exception as e:
         print(f"⚠️  Cloud SQL cleanup failed (continuing with destroy): {e}")
 
@@ -377,7 +328,7 @@ def cleanup_terraform_files(terraform_dir: Path):
     print("✅ Cleanup completed")
 
 
-def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None):
+def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None, verbose=False):
     """
     Run a subprocess command with a loading bar using rich.progress.
     Progress messages are based on the stack/resources from the YAML config if provided.
@@ -413,7 +364,18 @@ def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None):
 
     # Log output to file for debugging
     log_file = cwd / "terraform_apply.log"
-    
+
+    if verbose:
+        with open(log_file, "w") as f:
+            process = subprocess.Popen(
+                cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+            )
+            for line in iter(process.stdout.readline, ""):
+                print(line, end="", flush=True)
+                f.write(line)
+            returncode = process.wait()
+        return returncode
+
     with Progress(
         SpinnerColumn(),
         TextColumn("[progress.description]{task.description}"),
@@ -422,7 +384,7 @@ def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None):
         TimeElapsedColumn(),
     ) as progress:
         task = progress.add_task(resource_msgs[0], total=100)
-        
+
         # Open log file and keep it open until process completes
         f = open(log_file, "w")
         try:
@@ -441,7 +403,7 @@ def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None):
                     # If we exceed estimated time, slowly approach 95%
                     excess_time = elapsed - estimated_seconds
                     progress_percent = min(95, 85 + int(excess_time / 30))  # +1% per 30 seconds
-                
+
                 # Choose message based on progress
                 msg_idx = min(
                     int(progress_percent / (100 / (n_msgs - 1))), n_msgs - 2
@@ -451,17 +413,17 @@ def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None):
                     task, completed=progress_percent, description=message
                 )
                 time.sleep(1)
-            
+
             # Wait for process to fully complete and flush all output
             returncode = process.wait()
             f.flush()  # Ensure all output is written
-            
+
             # Only show 100% if returncode is 0 (success)
             if returncode == 0:
                 progress.update(task, completed=100, description=resource_msgs[-1])
             else:
                 progress.update(task, completed=progress_percent, description=f"⚠️ Terraform apply returned code {returncode}")
-            
+
             return returncode
         finally:
             f.close()  # Always close the file