lamia-cloud 0.1.0__py3-none-any.whl

lamia_cloud/__init__.py

@@ -0,0 +1,59 @@
+"""lamia-cloud — lamia cloud services package.
+
+Public API:
+    get_cloud_llm() -> CloudLLM
+    get_scheduler(project_root) -> CloudScheduler
+    is_on_cloud() -> bool
+    Types: CloudLLMRequest, CloudLLMResponse, CloudScheduleJob, CloudJobStatus
+"""
+from lamia_cloud.interfaces import CloudLLM, CloudScheduler
+from lamia_cloud.types import CloudLLMRequest, CloudLLMResponse, CloudScheduleJob, CloudJobStatus
+from lamia_cloud.gcp import VertexLLM, is_on_gcp
+from lamia_cloud.loader import get_scheduler
+
+_llm_instance: CloudLLM = None
+
+
+def get_cloud_llm(region: str = "") -> CloudLLM:
+    """Return the cloud LLM instance.
+
+    Only GCP is supported, so we return VertexLLM directly without
+    conditional provider checking.
+    """
+    global _llm_instance
+    if _llm_instance is None:
+        _llm_instance = VertexLLM(region=region or _detect_region())
+    return _llm_instance
+
+
+def _detect_region() -> str:
+    """Auto-detect GCP region from Cloud Run metadata server."""
+    import urllib.request
+    try:
+        req = urllib.request.Request(
+            "http://metadata.google.internal/computeMetadata/v1/instance/region",
+            headers={"Metadata-Flavor": "Google"},
+        )
+        resp = urllib.request.urlopen(req, timeout=2)
+        full = resp.read().decode().strip()
+        return full.rsplit("/", 1)[-1]
+    except Exception:
+        return "us-central1"
+
+
+def is_on_cloud() -> bool:
+    """Check if running in a cloud environment."""
+    return is_on_gcp()
+
+
+__all__ = [
+    "CloudLLM",
+    "CloudScheduler",
+    "CloudLLMRequest",
+    "CloudLLMResponse",
+    "CloudScheduleJob",
+    "CloudJobStatus",
+    "get_cloud_llm",
+    "get_scheduler",
+    "is_on_cloud",
+]

lamia_cloud/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

lamia_cloud/gcp/__init__.py

@@ -0,0 +1,10 @@
+"""GCP cloud backend — scheduler, LLM, and deployment pipeline."""
+
+from lamia_cloud.gcp.scheduler import GCPCloudScheduler
+from lamia_cloud.gcp.vertex import VertexLLM, is_on_gcp
+
+__all__ = [
+    "GCPCloudScheduler",
+    "VertexLLM",
+    "is_on_gcp",
+]

lamia_cloud/gcp/deployer.py

@@ -0,0 +1,366 @@
+"""Deploys a lamia project to Cloud Run via Cloud Build.
+
+Flow:
+1. Package the .lm script + project files into a staging directory
+2. Add Dockerfile + main.py handler + requirements.txt
+3. Upload to GCS as source tarball
+4. Submit Cloud Build to build the container
+5. Deploy the container to Cloud Run (with Vertex AI IAM for LLM access)
+6. Return the Cloud Run service URL
+
+LLM authentication uses Vertex AI — the Cloud Run service account gets
+roles/aiplatform.user, so no API keys are needed at runtime.
+"""
+
+import io
+import logging
+import shutil
+import tarfile
+import tempfile
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# templates/ lives at the lamia_cloud package root, one level above this gcp/ subpackage
+TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
+
+
+def _service_name(schedule_id: str) -> str:
+    return f"lamia-{schedule_id}"
+
+
+def _image_name(project_id: str, schedule_id: str) -> str:
+    import time
+    ts = int(time.time())
+    return f"gcr.io/{project_id}/lamia-{schedule_id}:{ts}"
+
+
+def _collect_project_files(project_root: Path) -> list[Path]:
+    """Collect .lm files, config.yaml, and supporting Python files from the project.
+
+    SECURITY: .env files are explicitly excluded — secrets must never be baked
+    into Docker image layers. API keys are injected via Cloud Run env vars at
+    deploy time (see deploy_cloud_run).
+    """
+    files = []
+    for pattern in ("*.lm", "*.py", "*.yaml", "*.yml", "*.json", "*.txt", "*.csv"):
+        files.extend(project_root.glob(pattern))
+    files = [f for f in files if f.name != ".env"]
+    for subdir in project_root.iterdir():
+        if subdir.is_dir() and not subdir.name.startswith("."):
+            for pattern in ("**/*.lm", "**/*.py", "**/*.yaml", "**/*.json"):
+                files.extend(subdir.glob(pattern))
+    return files
+
+
+def package_deployment(
+    project_root: Path,
+    script_name: str,
+    schedule_id: str,
+) -> Path:
+    """Create a staging directory with everything needed for the Cloud Build."""
+    staging = Path(tempfile.mkdtemp(prefix="lamia-deploy-"))
+
+    project_dest = staging / "project"
+    project_dest.mkdir()
+    for f in _collect_project_files(project_root):
+        rel = f.relative_to(project_root)
+        dest = project_dest / rel
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(f, dest)
+
+    shutil.copy2(TEMPLATES_DIR / "Dockerfile", staging / "Dockerfile")
+    shutil.copy2(TEMPLATES_DIR / "main.py", staging / "main.py")
+
+    requirements = staging / "requirements.txt"
+    project_requirements = project_root / "requirements.txt"
+    if project_requirements.exists():
+        reqs = project_requirements.read_text()
+    else:
+        reqs = ""
+    if "lamia-lang" not in reqs:
+        reqs = "lamia-lang\n" + reqs
+    if "google-auth" not in reqs:
+        reqs += "google-auth\n"
+    requirements.write_text(reqs)
+
+    return staging
+
+
+def create_source_tarball(staging_dir: Path) -> bytes:
+    """Create a gzipped tarball from the staging directory."""
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+        for item in staging_dir.iterdir():
+            tar.add(item, arcname=item.name)
+    return buf.getvalue()
+
+
+def upload_source(project_id: str, tarball: bytes, schedule_id: str) -> str:
+    """Upload source tarball to GCS and return the gs:// URI."""
+    from google.cloud import storage
+
+    bucket_name = f"{project_id}_cloudbuild"
+    blob_name = f"lamia-source/{schedule_id}.tar.gz"
+
+    client = storage.Client(project=project_id)
+    bucket = client.bucket(bucket_name)
+    if not bucket.exists():
+        bucket = client.create_bucket(bucket_name, location="us")
+
+    blob = bucket.blob(blob_name)
+    blob.upload_from_string(tarball, content_type="application/gzip")
+
+    return f"gs://{bucket_name}/{blob_name}"
+
+
+def submit_build(
+    project_id: str,
+    source_uri: str,
+    image_name: str,
+) -> None:
+    """Submit a Cloud Build to build the container image."""
+    from google.cloud.devtools import cloudbuild_v1
+
+    client = cloudbuild_v1.CloudBuildClient()
+
+    build = cloudbuild_v1.Build(
+        source=cloudbuild_v1.Source(
+            storage_source=cloudbuild_v1.StorageSource(
+                bucket=source_uri.split("/")[2],
+                object_="/".join(source_uri.split("/")[3:]),
+            )
+        ),
+        steps=[
+            cloudbuild_v1.BuildStep(
+                name="gcr.io/cloud-builders/docker",
+                args=["build", "-t", image_name, "."],
+            )
+        ],
+        images=[image_name],
+    )
+
+    operation = client.create_build(project_id=project_id, build=build)
+    logger.info(f"Cloud Build submitted, waiting for completion...")
+    result = operation.result(timeout=600)
+
+    if result.status != cloudbuild_v1.Build.Status.SUCCESS:
+        raise RuntimeError(
+            f"Cloud Build failed with status {result.status.name}: "
+            f"{result.status_detail}"
+        )
+    logger.info(f"Cloud Build succeeded: {image_name}")
+
+
+def deploy_cloud_run(
+    project_id: str,
+    location: str,
+    service_name: str,
+    image_name: str,
+    script_name: str,
+) -> str:
+    """Deploy (or update) a Cloud Run service. Returns the service URL.
+
+    LLM auth is handled by Vertex AI — the service account gets
+    roles/aiplatform.user so no API keys are injected.
+    """
+    from google.cloud import run_v2
+
+    client = run_v2.ServicesClient()
+    parent = f"projects/{project_id}/locations/{location}"
+    full_name = f"{parent}/services/{service_name}"
+
+    service_account = _ensure_service_account(project_id)
+
+    env_vars = [
+        run_v2.EnvVar(name="LAMIA_SCRIPT", value=script_name),
+        run_v2.EnvVar(name="GOOGLE_CLOUD_PROJECT", value=project_id),
+    ]
+
+    container = run_v2.Container(
+        image=image_name,
+        env=env_vars,
+        resources=run_v2.ResourceRequirements(
+            limits={"memory": "512Mi", "cpu": "1"},
+        ),
+    )
+
+    service = run_v2.Service(
+        template=run_v2.RevisionTemplate(
+            containers=[container],
+            service_account=service_account,
+            max_instance_request_concurrency=1,
+            timeout={"seconds": 540},
+        ),
+        ingress=run_v2.IngressTraffic.INGRESS_TRAFFIC_INTERNAL_ONLY,
+    )
+
+    from google.api_core.exceptions import NotFound
+
+    service.name = full_name
+
+    try:
+        operation = client.update_service(service=service)
+        result = operation.result(timeout=300)
+    except NotFound:
+        service.name = ""
+        operation = client.create_service(
+            parent=parent,
+            service=service,
+            service_id=service_name,
+        )
+        result = operation.result(timeout=300)
+
+    url = result.uri
+    logger.info(f"Cloud Run deployed: {url}")
+
+    _allow_scheduler_invocation(project_id, location, service_name)
+
+    return url
+
+
+def _ensure_service_account(project_id: str) -> str:
+    """Create lamia-runner service account with required permissions.
+
+    Grants:
+    - roles/aiplatform.user — Vertex AI model access
+    - Cloud Scheduler agent gets token creator on lamia-runner (for OIDC signing)
+    """
+    from google.cloud import iam_admin_v1
+    from google.cloud import resourcemanager_v3
+    from google.iam.v1 import policy_pb2
+
+    sa_email = f"lamia-runner@{project_id}.iam.gserviceaccount.com"
+    iam_client = iam_admin_v1.IAMClient()
+
+    try:
+        iam_client.get_service_account(
+            request={"name": f"projects/{project_id}/serviceAccounts/{sa_email}"}
+        )
+    except Exception as e:
+        if "NOT_FOUND" in str(e):
+            iam_client.create_service_account(
+                request={
+                    "name": f"projects/{project_id}",
+                    "account_id": "lamia-runner",
+                    "service_account": {"display_name": "Lamia Cloud Runner"},
+                }
+            )
+            logger.info(f"Created service account: {sa_email}")
+        else:
+            raise
+
+    rm_client = resourcemanager_v3.ProjectsClient()
+    resource = f"projects/{project_id}"
+    policy = rm_client.get_iam_policy(request={"resource": resource})
+
+    project_number = _get_project_number(project_id)
+    scheduler_sa = f"service-{project_number}@gcp-sa-cloudscheduler.iam.gserviceaccount.com"
+    member = f"serviceAccount:{sa_email}"
+
+    required_bindings = {
+        "roles/aiplatform.user": [member],
+        "roles/iam.serviceAccountTokenCreator": [f"serviceAccount:{scheduler_sa}"],
+    }
+
+    changed = False
+    for role, members in required_bindings.items():
+        for m in members:
+            already = any(
+                b.role == role and m in b.members for b in policy.bindings
+            )
+            if not already:
+                policy.bindings.append(
+                    policy_pb2.Binding(role=role, members=[m])
+                )
+                logger.info(f"Granted {role} to {m}")
+                changed = True
+
+    if changed:
+        rm_client.set_iam_policy(request={"resource": resource, "policy": policy})
+
+    return sa_email
+
+
+def _allow_scheduler_invocation(project_id: str, location: str, service_name: str) -> None:
+    """Grant Cloud Scheduler permission to invoke the Cloud Run service."""
+    from google.cloud import run_v2
+    from google.iam.v1 import iam_policy_pb2, policy_pb2
+
+    client = run_v2.ServicesClient()
+    resource = f"projects/{project_id}/locations/{location}/services/{service_name}"
+
+    try:
+        policy = client.get_iam_policy(request={"resource": resource})
+    except Exception:
+        policy = policy_pb2.Policy()
+
+    invoker_role = "roles/run.invoker"
+    scheduler_sa = f"service-{_get_project_number(project_id)}@gcp-sa-cloudscheduler.iam.gserviceaccount.com"
+    member = f"serviceAccount:{scheduler_sa}"
+
+    for binding in policy.bindings:
+        if binding.role == invoker_role and member in binding.members:
+            return
+
+    policy.bindings.append(
+        policy_pb2.Binding(role=invoker_role, members=[member])
+    )
+    client.set_iam_policy(request={"resource": resource, "policy": policy})
+
+
+def _get_project_number(project_id: str) -> str:
+    """Get the project number from project ID."""
+    from google.cloud import resourcemanager_v3
+
+    client = resourcemanager_v3.ProjectsClient()
+    project = client.get_project(name=f"projects/{project_id}")
+    return project.name.split("/")[1]
+
+
+def deploy(
+    project_id: str,
+    location: str,
+    project_root: Path,
+    script_name: str,
+    schedule_id: str,
+) -> str:
+    """Full deploy pipeline. Returns the Cloud Run service URL."""
+    service_name = _service_name(schedule_id)
+    image = _image_name(project_id, schedule_id)
+
+    logger.info(f"Packaging {script_name} for deployment...")
+    staging = package_deployment(project_root, script_name, schedule_id)
+
+    try:
+        logger.info("Creating source tarball...")
+        tarball = create_source_tarball(staging)
+
+        logger.info("Uploading source to GCS...")
+        source_uri = upload_source(project_id, tarball, schedule_id)
+
+        logger.info("Submitting Cloud Build...")
+        submit_build(project_id, source_uri, image)
+
+        logger.info("Deploying to Cloud Run with Vertex AI access...")
+        url = deploy_cloud_run(project_id, location, service_name, image, script_name)
+
+        return url
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+
+
+def teardown(project_id: str, location: str, schedule_id: str) -> None:
+    """Remove the Cloud Run service for a schedule."""
+    from google.cloud import run_v2
+
+    client = run_v2.ServicesClient()
+    service_name = _service_name(schedule_id)
+    full_name = f"projects/{project_id}/locations/{location}/services/{service_name}"
+
+    try:
+        client.delete_service(name=full_name)
+        logger.info(f"Deleted Cloud Run service: {service_name}")
+    except Exception as e:
+        if "NOT_FOUND" not in str(e):
+            raise

lamia_cloud/gcp/scheduler.py

@@ -0,0 +1,213 @@
+"""GCP Cloud Scheduler backend.
+
+Orchestrates: Cloud Build -> Cloud Run -> Cloud Scheduler.
+The user only provides project_id and location. Everything else is automated.
+"""
+
+import json
+import logging
+import time
+from typing import Optional
+
+from lamia_cloud.interfaces import CloudScheduler
+from lamia_cloud.types import CloudScheduleJob, CloudJobStatus
+from lamia_cloud.gcp.deployer import deploy, teardown
+
+logger = logging.getLogger(__name__)
+
+
+def _enable_apis(project_id: str) -> None:
+    """Enable all required GCP APIs automatically.
+
+    Service Usage API must be enabled first (bootstraps the rest).
+    Most GCP projects have it enabled by default.
+    """
+    try:
+        from google.cloud import service_usage_v1
+        client = service_usage_v1.ServiceUsageClient()
+        apis = [
+            "serviceusage.googleapis.com",
+            "cloudscheduler.googleapis.com",
+            "cloudbuild.googleapis.com",
+            "run.googleapis.com",
+            "storage.googleapis.com",
+            "aiplatform.googleapis.com",
+            "iam.googleapis.com",
+        ]
+        for api in apis:
+            service_name = f"projects/{project_id}/services/{api}"
+            try:
+                client.enable_service(request={"name": service_name})
+            except Exception as e:
+                if "SERVICE_DISABLED" in str(e) and "serviceusage" in api:
+                    logger.warning(
+                        f"Service Usage API not enabled. Run once:\n"
+                        f"  gcloud services enable serviceusage.googleapis.com "
+                        f"--project={project_id}"
+                    )
+                    return
+    except ImportError:
+        pass
+
+
+class GCPCloudScheduler(CloudScheduler):
+    """GCP Cloud Scheduler backend with automatic deployment."""
+
+    def __init__(self, *, project_id: str, location: str):
+        self.project_id = project_id
+        self.location = location
+        self._parent = f"projects/{project_id}/locations/{location}"
+
+    @classmethod
+    def name(cls) -> str:
+        return "gcp-cloud"
+
+    @classmethod
+    def from_config(cls, cloud_cfg: dict) -> "GCPCloudScheduler":
+        """Create instance from config.yaml cloud section."""
+        project_id = cloud_cfg.get("project_id")
+        if not project_id:
+            raise ValueError("cloud.project_id is required in config.yaml.")
+
+        location = cloud_cfg.get("location", "us-central1")
+
+        _enable_apis(project_id)
+        return cls(project_id=project_id, location=location)
+
+    def _scheduler_client(self):
+        from google.cloud import scheduler_v1
+        return scheduler_v1.CloudSchedulerClient()
+
+    def _job_name(self, job: CloudScheduleJob) -> str:
+        return f"{self._parent}/jobs/lamia-{job.schedule_id}"
+
+    def _build_scheduler_job(self, job: CloudScheduleJob, target_url: str):
+        from google.cloud import scheduler_v1
+
+        schedule = job.cron
+        if schedule == "@reboot":
+            schedule = "0 * * * *"
+
+        body = json.dumps({
+            "schedule_id": job.schedule_id,
+            "script": job.script,
+        }).encode()
+
+        return scheduler_v1.Job(
+            name=self._job_name(job),
+            schedule=schedule,
+            time_zone="UTC",
+            http_target=scheduler_v1.HttpTarget(
+                uri=target_url,
+                http_method=scheduler_v1.HttpMethod.POST,
+                headers={"Content-Type": "application/json"},
+                body=body,
+                oidc_token=scheduler_v1.OidcToken(
+                    service_account_email=(
+                        f"lamia-runner@{self.project_id}.iam.gserviceaccount.com"
+                    ),
+                    audience=target_url,
+                ),
+            ),
+        )
+
+    def install(self, job: CloudScheduleJob, lamia_bin: str) -> None:
+        """Deploy the script to Cloud Run and create a Cloud Scheduler trigger."""
+        logger.info(f"Deploying {job.script} to cloud...")
+
+        service_url = deploy(
+            project_id=self.project_id,
+            location=self.location,
+            project_root=job.project_root,
+            script_name=job.script,
+            schedule_id=job.schedule_id,
+        )
+
+        client = self._scheduler_client()
+        scheduler_job = self._build_scheduler_job(job, service_url)
+
+        from google.api_core.exceptions import AlreadyExists, NotFound
+
+        for attempt in range(5):
+            try:
+                client.create_job(parent=self._parent, job=scheduler_job)
+                logger.info(f"Created cloud schedule: {job.script} ({job.cron})")
+                break
+            except AlreadyExists:
+                client.update_job(job=scheduler_job)
+                logger.info(f"Updated cloud schedule: {job.script} ({job.cron})")
+                break
+            except NotFound:
+                if attempt == 4:
+                    raise RuntimeError(
+                        f"Cloud Scheduler location '{self.location}' not ready after retries. "
+                        f"The API may still be propagating — wait a minute and retry."
+                    )
+                logger.info(f"Cloud Scheduler not ready yet, retrying in 15s...")
+                time.sleep(15)
+
+    def uninstall(self, job: CloudScheduleJob) -> None:
+        """Remove both the Cloud Scheduler job and the Cloud Run service."""
+        client = self._scheduler_client()
+        from google.api_core.exceptions import Aborted, NotFound
+
+        for attempt in range(5):
+            try:
+                client.delete_job(name=self._job_name(job))
+                logger.info(f"Deleted cloud schedule: {job.script}")
+                break
+            except NotFound:
+                break
+            except Aborted:
+                if attempt == 4:
+                    raise
+                time.sleep(10)
+
+        teardown(self.project_id, self.location, job.schedule_id)
+
+    def is_installed(self, job: CloudScheduleJob) -> bool:
+        client = self._scheduler_client()
+        try:
+            client.get_job(name=self._job_name(job))
+            return True
+        except Exception:
+            return False
+
+    def get_status(self, job: CloudScheduleJob) -> CloudJobStatus:
+        client = self._scheduler_client()
+        try:
+            cloud_job = client.get_job(name=self._job_name(job))
+            from google.cloud.scheduler_v1 import Job
+            if cloud_job.state == Job.State.ENABLED:
+                return CloudJobStatus.ACTIVE
+            return CloudJobStatus.INACTIVE
+        except Exception:
+            return CloudJobStatus.UNKNOWN
+
+    def pause(self, job: CloudScheduleJob) -> None:
+        """Pause the Cloud Scheduler job (stops triggering)."""
+        client = self._scheduler_client()
+        client.pause_job(name=self._job_name(job))
+        logger.info(f"Paused cloud schedule: {job.script}")
+
+    def resume(self, job: CloudScheduleJob) -> None:
+        """Resume a paused Cloud Scheduler job."""
+        client = self._scheduler_client()
+        client.resume_job(name=self._job_name(job))
+        logger.info(f"Resumed cloud schedule: {job.script}")
+
+    def get_installed_config(self, job: CloudScheduleJob) -> Optional[dict]:
+        client = self._scheduler_client()
+        try:
+            cloud_job = client.get_job(name=self._job_name(job))
+            return {
+                "schedule": cloud_job.schedule,
+                "state": cloud_job.state.name,
+                "last_attempt_time": (
+                    cloud_job.last_attempt_time.isoformat()
+                    if cloud_job.last_attempt_time
+                    else None
+                ),
+            }
+        except Exception:
+            return None