duckless 0.1.0__py3-none-any.whl

duckless/__init__.py

@@ -0,0 +1,5 @@
+"""DuckLess: serverless DuckDB on GCP.
+
+Submit SQL or your own code; DuckLess runs it on a right-sized Compute Engine VM
+(Cloud Batch) in your project, with GCS access through ADC, then tears it down.
+"""

duckless/adapters/__init__.py

@@ -0,0 +1 @@
+"""Driven adapters: GCP implementations of duckless.ports."""

duckless/adapters/batch.py

@@ -0,0 +1,148 @@
+"""Executor on Cloud Batch: one task, one VM per job, deleted when the job ends."""
+
+from google.api_core.exceptions import NotFound
+from google.cloud import batch_v1
+
+from duckless.core.errors import JobNotFoundError
+from duckless.core.job import JobEvent, JobKind, JobSpec, JobState, JobStatus
+from duckless.core.machine import LOCAL_SSD_GB
+from duckless.settings import Settings
+
+SCRATCH_PATH = "/mnt/disks/scratch"
+SCRATCH_DEVICE = "scratch"
+# Runner exit codes (job failed / bad usage): fail at once, keep retries for infra failures
+# such as a Spot preemption (exit 50001). Batch allows a single lifecycle policy per task.
+RUNNER_EXIT_CODES = (1, 2)
+MAX_RETRIES = 2
+
+_STATES = {
+    batch_v1.JobStatus.State.QUEUED: JobState.QUEUED,
+    batch_v1.JobStatus.State.SCHEDULED: JobState.SCHEDULED,
+    batch_v1.JobStatus.State.RUNNING: JobState.RUNNING,
+    batch_v1.JobStatus.State.SUCCEEDED: JobState.SUCCEEDED,
+    batch_v1.JobStatus.State.FAILED: JobState.FAILED,
+    batch_v1.JobStatus.State.CANCELLED: JobState.CANCELLED,
+}
+
+
+# ---------- pure ----------
+
+
+def container_invocation(spec: JobSpec) -> tuple[str, list[str]]:
+    """(entrypoint, args). Runner jobs keep the image entrypoint (`python -m duckless_runtime`) and
+    pass `sql|py <uri>`; a command job replaces the entrypoint, so `-- dbt build` runs dbt itself."""
+    if spec.kind is JobKind.COMMAND:
+        return spec.command[0], list(spec.command[1:])
+    return "", list(spec.runner_args)
+
+
+def build_job(spec: JobSpec, settings: Settings) -> batch_v1.Job:
+    has_scratch = spec.local_ssd_count > 0
+    entrypoint, args = container_invocation(spec)
+    # Local SSD is mounted by Batch on the host as root; open it to the non-root runner user.
+    prepare_scratch = batch_v1.Runnable(
+        script=batch_v1.Runnable.Script(text=f"mkdir -p {SCRATCH_PATH} && chmod 1777 {SCRATCH_PATH}")
+    )
+    runner = batch_v1.Runnable(
+        # Task volumes are bind-mounted into the container at the same path by default.
+        container=batch_v1.Runnable.Container(image_uri=spec.image, entrypoint=entrypoint, commands=args),
+        environment=batch_v1.Environment(variables={**dict(spec.env), "GOOGLE_CLOUD_PROJECT": settings.project}),
+    )
+    task = batch_v1.TaskSpec(
+        runnables=[prepare_scratch, runner] if has_scratch else [runner],
+        volumes=[batch_v1.Volume(device_name=SCRATCH_DEVICE, mount_path=SCRATCH_PATH)] if has_scratch else [],
+        max_run_duration=f"{spec.max_run_seconds}s",
+        max_retry_count=MAX_RETRIES,
+        lifecycle_policies=[
+            batch_v1.LifecyclePolicy(
+                action=batch_v1.LifecyclePolicy.Action.FAIL_TASK,
+                action_condition=batch_v1.LifecyclePolicy.ActionCondition(exit_codes=list(RUNNER_EXIT_CODES)),
+            )
+        ],
+    )
+    policy = batch_v1.AllocationPolicy.InstancePolicy(
+        machine_type=spec.machine.name,
+        provisioning_model=(
+            batch_v1.AllocationPolicy.ProvisioningModel.SPOT
+            if spec.spot
+            else batch_v1.AllocationPolicy.ProvisioningModel.STANDARD
+        ),
+        disks=[
+            batch_v1.AllocationPolicy.AttachedDisk(
+                new_disk=batch_v1.AllocationPolicy.Disk(type_="local-ssd", size_gb=LOCAL_SSD_GB * spec.local_ssd_count),
+                device_name=SCRATCH_DEVICE,
+            )
+        ]
+        if has_scratch
+        else [],
+    )
+    allocation = batch_v1.AllocationPolicy(
+        # Any zone of the region: Spot + highmem + local SSD stocks out zone by zone.
+        location=batch_v1.AllocationPolicy.LocationPolicy(allowed_locations=[f"regions/{settings.region}"]),
+        instances=[batch_v1.AllocationPolicy.InstancePolicyOrTemplate(policy=policy)],
+        service_account=batch_v1.ServiceAccount(email=settings.service_account),
+        network=batch_v1.AllocationPolicy.NetworkPolicy(
+            network_interfaces=[
+                batch_v1.AllocationPolicy.NetworkInterface(
+                    network=settings.network,
+                    subnetwork=settings.subnetwork,
+                    no_external_ip_address=not settings.external_ip,
+                )
+            ]
+        ),
+    )
+    return batch_v1.Job(
+        task_groups=[batch_v1.TaskGroup(task_spec=task, task_count=1)],
+        allocation_policy=allocation,
+        logs_policy=batch_v1.LogsPolicy(destination=batch_v1.LogsPolicy.Destination.CLOUD_LOGGING),
+        labels={"app": "duckless", "duckless-kind": spec.kind.value},
+    )
+
+
+def status_from_job(job: batch_v1.Job) -> JobStatus:
+    policy = job.allocation_policy.instances[0].policy
+    return JobStatus(
+        job_id=job.name.rsplit("/", 1)[-1],
+        uid=job.uid,
+        state=_STATES.get(job.status.state, JobState.UNKNOWN),
+        machine=policy.machine_type,
+        spot=policy.provisioning_model == batch_v1.AllocationPolicy.ProvisioningModel.SPOT,
+        created_at=job.create_time,
+        run_seconds=job.status.run_duration.total_seconds() if job.status.run_duration else None,
+        events=tuple(
+            JobEvent(at=e.event_time, description=e.description.split(" for job ")[0]) for e in job.status.status_events
+        ),
+    )
+
+
+# ---------- adapter ----------
+
+
+class BatchExecutor:
+    def __init__(self, client: batch_v1.BatchServiceClient, settings: Settings) -> None:
+        self._client = client
+        self._settings = settings
+
+    def _parent(self) -> str:
+        return f"projects/{self._settings.project}/locations/{self._settings.region}"
+
+    def _name(self, job_id: str) -> str:
+        return f"{self._parent()}/jobs/{job_id}"
+
+    def submit(self, spec: JobSpec) -> JobStatus:
+        created = self._client.create_job(
+            parent=self._parent(), job_id=spec.job_id, job=build_job(spec, self._settings)
+        )
+        return status_from_job(created)
+
+    def get(self, job_id: str) -> JobStatus:
+        try:
+            return status_from_job(self._client.get_job(name=self._name(job_id)))
+        except NotFound as e:
+            raise JobNotFoundError(job_id) from e
+
+    def cancel(self, job_id: str) -> None:
+        try:
+            self._client.cancel_job(name=self._name(job_id))
+        except NotFound as e:
+            raise JobNotFoundError(job_id) from e

duckless/adapters/cloud_logging.py

@@ -0,0 +1,45 @@
+"""Runner logs from Cloud Logging: Batch writes task stdout to `batch_task_logs`, labelled by job uid."""
+
+from datetime import datetime
+
+from google.cloud import logging as cloud_logging
+
+from duckless.core.job import LogLine
+
+TASK_LOG = "batch_task_logs"
+
+
+def log_filter(project: str, job_uid: str, since: datetime | None) -> str:
+    clauses = (
+        f'logName="projects/{project}/logs/{TASK_LOG}"',
+        f'labels.job_uid="{job_uid}"',
+        *((f'timestamp>"{since.isoformat()}"',) if since else ()),
+    )
+    return " AND ".join(clauses)
+
+
+def to_line(entry: cloud_logging.LogEntry) -> LogLine:
+    """The runner logs JSON lines (jsonPayload); anything else (image pull, crash) is plain text."""
+    payload = entry.payload if isinstance(entry.payload, dict) else {"message": str(entry.payload)}
+    fields = {k: v for k, v in payload.items() if k not in {"message", "severity", "time"}}
+    return LogLine(
+        at=entry.timestamp,
+        severity=entry.severity or "DEFAULT",
+        message=str(payload.get("message", "")),
+        fields=fields,
+    )
+
+
+class CloudLoggingLogReader:
+    def __init__(self, client: cloud_logging.Client, project: str) -> None:
+        self._client = client
+        self._project = project
+
+    def read(self, job_uid: str, since: datetime | None = None, limit: int = 200) -> tuple[LogLine, ...]:
+        entries = self._client.list_entries(
+            resource_names=[f"projects/{self._project}"],
+            filter_=log_filter(self._project, job_uid, since),
+            order_by=cloud_logging.ASCENDING,
+            max_results=limit,
+        )
+        return tuple(map(to_line, entries))

duckless/adapters/compute_quotas.py

@@ -0,0 +1,15 @@
+from collections.abc import Mapping
+
+from google.cloud import compute_v1
+
+from duckless.core.quota import Quota
+
+
+class ComputeQuotaReader:
+    def __init__(self, client: compute_v1.RegionsClient, project: str) -> None:
+        self._client = client
+        self._project = project
+
+    def regional_quotas(self, region: str) -> Mapping[str, Quota]:
+        regional = self._client.get(project=self._project, region=region)
+        return {q.metric: Quota(usage=q.usage, limit=q.limit) for q in regional.quotas}

duckless/adapters/gcp_bootstrap.py

@@ -0,0 +1,102 @@
+"""InfraBootstrap with the caller's credentials: plain REST for the few one-off calls, the
+storage client for the staging bucket."""
+
+import time
+from pathlib import Path
+
+from google.auth.transport.requests import AuthorizedSession
+from google.cloud import storage
+
+from duckless.core.infra import with_bindings, without_bindings
+
+SERVICE_USAGE = "https://serviceusage.googleapis.com/v1"
+IAM = "https://iam.googleapis.com/v1"
+RESOURCE_MANAGER = "https://cloudresourcemanager.googleapis.com/v3"
+SKIPPED_PARTS = frozenset({".terraform", ".terraform.lock.hcl", "__pycache__"})
+OPERATION_POLL_SECONDS = 3
+
+
+def module_files(local_dir: Path) -> list[Path]:
+    """Files of the Terraform module, without local Terraform state or caches."""
+    return sorted(
+        p
+        for p in local_dir.rglob("*")
+        if p.is_file() and not SKIPPED_PARTS.intersection(p.relative_to(local_dir).parts)
+    )
+
+
+class GcpInfraBootstrap:
+    def __init__(self, session: AuthorizedSession, storage_client: storage.Client) -> None:
+        self._session = session
+        self._storage = storage_client
+
+    def _call(self, method: str, url: str, **kwargs) -> dict:
+        response = self._session.request(method, url, **kwargs)
+        response.raise_for_status()
+        return response.json() if response.content else {}
+
+    def _wait(self, base: str, operation: dict) -> None:
+        while not operation.get("done"):
+            time.sleep(OPERATION_POLL_SECONDS)
+            operation = self._call("GET", f"{base}/{operation['name']}")
+        if "error" in operation:
+            raise RuntimeError(f"operation {operation['name']} failed: {operation['error']}")
+
+    def enable_apis(self, project: str, apis: tuple[str, ...]) -> None:
+        operation = self._call(
+            "POST", f"{SERVICE_USAGE}/projects/{project}/services:batchEnable", json={"serviceIds": list(apis)}
+        )
+        self._wait(SERVICE_USAGE, operation)
+
+    def ensure_service_account(self, project: str, account_id: str, display_name: str) -> str:
+        email = f"{account_id}@{project}.iam.gserviceaccount.com"
+        existing = self._session.get(f"{IAM}/projects/{project}/serviceAccounts/{email}")
+        if existing.status_code == 404:
+            self._call(
+                "POST",
+                f"{IAM}/projects/{project}/serviceAccounts",
+                json={"accountId": account_id, "serviceAccount": {"displayName": display_name}},
+            )
+        else:
+            existing.raise_for_status()
+        return email
+
+    def _update_policy(self, project: str, change) -> bool:
+        resource = f"{RESOURCE_MANAGER}/projects/{project}"
+        policy = self._call("POST", f"{resource}:getIamPolicy", json={"options": {"requestedPolicyVersion": 3}})
+        updated, changed = change(policy)
+        if changed:
+            # The etag in `updated` makes a concurrent change fail instead of being overwritten.
+            self._call("POST", f"{resource}:setIamPolicy", json={"policy": updated})
+        return changed
+
+    def grant_project_roles(self, project: str, member: str, roles: tuple[str, ...]) -> bool:
+        return self._update_policy(project, lambda policy: with_bindings(policy, member, roles))
+
+    def revoke_project_roles(self, project: str, member: str, roles: tuple[str, ...]) -> None:
+        self._update_policy(project, lambda policy: without_bindings(policy, member, roles))
+
+    def delete_service_account(self, project: str, email: str) -> None:
+        response = self._session.delete(f"{IAM}/projects/{project}/serviceAccounts/{email}")
+        if response.status_code != 404:
+            response.raise_for_status()
+
+    def delete_bucket(self, bucket: str) -> None:
+        existing = self._storage.lookup_bucket(bucket)
+        if existing is not None:
+            existing.delete(force=True)
+
+    def ensure_bucket(self, project: str, region: str, bucket: str) -> None:
+        if self._storage.lookup_bucket(bucket) is not None:
+            return
+        new = self._storage.bucket(bucket)
+        new.iam_configuration.uniform_bucket_level_access_enabled = True
+        new.iam_configuration.public_access_prevention = "enforced"
+        new.labels = {"app": "duckless"}
+        self._storage.create_bucket(new, project=project, location=region)
+
+    def upload_directory(self, local_dir: Path, bucket: str, prefix: str) -> str:
+        target = self._storage.bucket(bucket)
+        for path in module_files(local_dir):
+            target.blob(f"{prefix}/{path.relative_to(local_dir).as_posix()}").upload_from_filename(str(path))
+        return f"gs://{bucket}/{prefix}"

duckless/adapters/gcs.py

@@ -0,0 +1,42 @@
+"""Work bucket layout: gs://<bucket>/runs/<job_id>/{<source>, metrics.json}."""
+
+import json
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from google.cloud import storage
+
+RUNS_PREFIX = "runs"
+METRICS_FILE = "metrics.json"
+
+
+def run_object(job_id: str, name: str) -> str:
+    return f"{RUNS_PREFIX}/{job_id}/{name}"
+
+
+def parse_metrics(raw: str) -> Mapping[str, Any] | None:
+    """The runner writes metrics with DuckDB's COPY … (FORMAT json): one object per line."""
+    first = next((line for line in raw.splitlines() if line.strip()), None)
+    return json.loads(first) if first else None
+
+
+class GcsArtifactStore:
+    def __init__(self, client: storage.Client, bucket: str) -> None:
+        self._bucket = client.bucket(bucket)
+
+    def upload_source(self, job_id: str, source: Path) -> str:
+        blob = self._bucket.blob(run_object(job_id, source.name))
+        blob.upload_from_filename(str(source))
+        return f"gs://{self._bucket.name}/{blob.name}"
+
+    def runner_env(self, job_id: str) -> Mapping[str, str]:
+        return {
+            "DUCKLESS_JOB_ID": job_id,
+            "DUCKLESS_BUCKET": self._bucket.name,
+            "DUCKLESS_METRICS_URI": f"gs://{self._bucket.name}/{run_object(job_id, METRICS_FILE)}",
+        }
+
+    def read_metrics(self, job_id: str) -> Mapping[str, Any] | None:
+        blob = self._bucket.blob(run_object(job_id, METRICS_FILE))
+        return parse_metrics(blob.download_as_text()) if blob.exists() else None

duckless/adapters/infra_manager.py

@@ -0,0 +1,113 @@
+"""InfraDeployer on Infrastructure Manager (Terraform run by Google, state kept in the project)."""
+
+import contextlib
+from collections.abc import Mapping
+from typing import Any
+
+from google.api_core.exceptions import GoogleAPICallError, NotFound
+from google.cloud import config_v1
+
+from duckless.core.infra import InfraStatus
+
+APPLY_TIMEOUT_SECONDS = 30 * 60
+LABELS = {"app": "duckless"}
+
+
+def deployment_name(project: str, region: str, deployment_id: str) -> str:
+    return f"projects/{project}/locations/{region}/deployments/{deployment_id}"
+
+
+def build_deployment(
+    name: str, source_uri: str, inputs: Mapping[str, Any], service_account: str
+) -> config_v1.Deployment:
+    return config_v1.Deployment(
+        name=name,
+        service_account=service_account
+        if service_account.startswith("projects/")
+        else f"projects/{name.split('/')[1]}/serviceAccounts/{service_account}",
+        terraform_blueprint=config_v1.TerraformBlueprint(
+            gcs_source=source_uri,
+            input_values={k: config_v1.TerraformVariable(input_value=v) for k, v in inputs.items()},
+        ),
+        labels=LABELS,
+    )
+
+
+def failure_detail(deployment: config_v1.Deployment, revision: config_v1.Revision | None) -> str:
+    """The most useful error Infra Manager kept: Terraform errors first, then the state detail."""
+    tf_errors = [e.error_description or e.resource_address for e in (revision.tf_errors if revision else ())]
+    tf_errors += [e.error_description or e.resource_address for e in deployment.tf_errors]
+    logs = revision.logs if revision and revision.logs else deployment.error_logs
+    parts = [
+        *tf_errors[:5],
+        deployment.state_detail or (revision.state_detail if revision else ""),
+        logs and f"logs: {logs}",
+    ]
+    return "\n".join(p for p in parts if p) or deployment.state.name
+
+
+class InfraManagerDeployer:
+    def __init__(self, client: config_v1.ConfigClient) -> None:
+        self._client = client
+
+    def _revision(self, deployment: config_v1.Deployment) -> config_v1.Revision | None:
+        return self._client.get_revision(name=deployment.latest_revision) if deployment.latest_revision else None
+
+    def _status(self, deployment: config_v1.Deployment) -> InfraStatus:
+        revision = self._revision(deployment)
+        outputs = {k: o.value for k, o in revision.apply_results.outputs.items()} if revision else {}
+        failed = deployment.state == config_v1.Deployment.State.FAILED
+        return InfraStatus(
+            deployment=deployment.name,
+            state=deployment.state.name,
+            outputs=outputs,
+            error=failure_detail(deployment, revision) if failed else None,
+        )
+
+    def get(self, project: str, region: str, deployment_id: str) -> InfraStatus | None:
+        try:
+            return self._status(self._client.get_deployment(name=deployment_name(project, region, deployment_id)))
+        except NotFound:
+            return None
+
+    def apply(
+        self,
+        project: str,
+        region: str,
+        deployment_id: str,
+        source_uri: str,
+        inputs: Mapping[str, Any],
+        service_account: str,
+    ) -> InfraStatus:
+        name = deployment_name(project, region, deployment_id)
+        deployment = build_deployment(name, source_uri, inputs, service_account)
+        exists = self.get(project, region, deployment_id) is not None
+        operation = (
+            self._client.update_deployment(deployment=deployment)
+            if exists
+            else self._client.create_deployment(
+                parent=f"projects/{project}/locations/{region}", deployment_id=deployment_id, deployment=deployment
+            )
+        )
+        # On failure the deployment carries the details, read below.
+        with contextlib.suppress(GoogleAPICallError):
+            operation.result(timeout=APPLY_TIMEOUT_SECONDS)
+        return self._status(self._client.get_deployment(name=name))
+
+    def destroy(self, project: str, region: str, deployment_id: str) -> InfraStatus:
+        name = deployment_name(project, region, deployment_id)
+        if self.get(project, region, deployment_id) is None:
+            return InfraStatus(name, "DELETED")
+        try:
+            # force: also delete the deployment's revisions (nested resources), else the call is refused.
+            self._client.delete_deployment(
+                request=config_v1.DeleteDeploymentRequest(
+                    name=name, force=True, delete_policy=config_v1.DeleteDeploymentRequest.DeletePolicy.DELETE
+                )
+            ).result(timeout=APPLY_TIMEOUT_SECONDS)
+        except GoogleAPICallError as e:
+            current = self.get(project, region, deployment_id)
+            return InfraStatus(
+                name, current.state if current else "UNKNOWN", error=current.error if current else str(e)
+            )
+        return InfraStatus(name, "DELETED")