py-serverless-kit 0.1.0__py3-none-any.whl

py_serverless_kit/providers/base.py

@@ -0,0 +1,310 @@
+"""Provider plug-in contract.
+
+A provider turns a validated :class:`ServiceConfig` plus a built artifact into
+a :class:`TerraformDoc`, and supplies the small *shim* files that sit at the
+root of the bundle and forward the platform's native entrypoint to
+``py_serverless_kit.runtime.adapters.<provider>``.
+
+Adding a new cloud = subclass :class:`Provider`, write an adapter module, and
+register it in ``providers/registry.py``. See CONTRIBUTING.md.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, ClassVar, Dict, FrozenSet, List, Optional
+
+from ..config.schema import FunctionConfig, ServiceConfig, WorkflowConfig
+from .terraform_doc import TerraformDoc
+
+# Capability flags ------------------------------------------------------------
+HTTP = "http"
+SCHEDULE = "schedule"
+QUEUE = "queue"
+STORAGE = "storage"
+TOPIC = "topic"
+WORKFLOW_NATIVE = "workflow_native"
+ALL_CAPABILITIES = (HTTP, SCHEDULE, QUEUE, STORAGE, TOPIC, WORKFLOW_NATIVE)
+EVENT_CAPABILITIES = frozenset({HTTP, SCHEDULE, QUEUE, STORAGE, TOPIC})
+
+ORCHESTRATOR_HANDLER = "py_serverless_kit.runtime.workflow.orchestrate"
+PREFERRED_RUNTIME = "python3.12"
+
+
+class ProviderError(Exception):
+    pass
+
+
+@dataclass
+class Artifact:
+    path: Path
+    sha256: str
+    sha256_b64: str
+    size: int
+    file_count: int = 0
+    build_dir: Optional[Path] = None
+
+
+@dataclass
+class Issue:
+    level: str  # "error" | "warning"
+    message: str
+    where: str = ""
+
+    def __str__(self) -> str:
+        loc = f"[{self.where}] " if self.where else ""
+        return f"{self.level.upper()}: {loc}{self.message}"
+
+
+@dataclass
+class RenderContext:
+    config: ServiceConfig
+    provider: Provider
+    root: Path
+    work_dir: Path
+    artifact: Artifact
+    region: str
+    workflow_engines: Dict[str, str] = field(default_factory=dict)
+
+    @property
+    def service(self) -> str:
+        return self.config.service
+
+    @property
+    def stage(self) -> str:
+        return self.config.provider.stage
+
+    @property
+    def options(self) -> Dict[str, Any]:
+        return self.config.provider.options
+
+    def opt(self, key: str, default: Any = None) -> Any:
+        return self.config.provider.options.get(key, default)
+
+    def physical_name(self, logical: str) -> str:
+        return self.provider.physical_name(self.service, self.stage, logical)
+
+    @property
+    def prefix(self) -> str:
+        return self.provider.physical_name(self.service, self.stage, "")
+
+    def tags(self, fn: Optional[FunctionConfig] = None) -> Dict[str, str]:
+        base = {
+            "pskit-service": self.service,
+            "pskit-stage": self.stage,
+            "managed-by": "py-serverless-kit",
+        }
+        base.update(self.config.provider.tags)
+        if fn is not None:
+            base.update(fn.tags)
+        return base
+
+    def environment(self, name: str, fn: FunctionConfig) -> Dict[str, str]:
+        env = {
+            "PSKIT_PROVIDER": self.provider.name,
+            "PSKIT_SERVICE": self.service,
+            "PSKIT_STAGE": self.stage,
+            "PSKIT_REGION": self.region,
+            "PSKIT_FUNCTION": name,
+            "PSKIT_HANDLER": fn.handler,
+        }
+        env.update(self.config.provider.environment)
+        env.update(fn.environment)
+        return env
+
+    def function_items(self) -> List[tuple]:
+        return sorted(self.config.functions.items())
+
+
+def stable_suffix(*parts: str, length: int = 6) -> str:
+    # Naming only, not security: keeps generated names short, stable and unique.
+    return hashlib.sha1("/".join(parts).encode(), usedforsecurity=False).hexdigest()[:length]
+
+
+class Provider(ABC):
+    name: ClassVar[str]
+    display_name: ClassVar[str]
+    tier: ClassVar[str] = "beta"  # stable | beta | experimental
+    terraform_source: ClassVar[str]
+    terraform_version: ClassVar[Optional[str]] = None
+    terraform_local_name: ClassVar[Optional[str]] = None
+    default_region: ClassVar[str] = ""
+    capabilities: ClassVar[FrozenSet[str]] = frozenset({HTTP})
+    max_name_length: ClassVar[int] = 64
+    name_pattern: ClassVar[str] = r"[^a-z0-9-]"
+    runtimes: ClassVar[Dict[str, str]] = {}
+    vendor_dependencies: ClassVar[bool] = True
+    extra_requirements: ClassVar[List[str]] = []
+    max_timeout: ClassVar[int] = 900
+    notes: ClassVar[str] = ""
+
+    # ------------------------------------------------------------ naming
+    def physical_name(self, service: str, stage: str, logical: str) -> str:
+        raw = "-".join(p for p in (service, stage, logical) if p)
+        name = re.sub(self.name_pattern, "-", raw.lower()).strip("-")
+        name = re.sub(r"-{2,}", "-", name)
+        if len(name) > self.max_name_length:
+            digest = stable_suffix(raw)
+            name = name[: self.max_name_length - len(digest) - 1].rstrip("-") + "-" + digest
+        return name
+
+    @property
+    def default_runtime(self) -> str:
+        """Runtime used when the config does not pin one: python3.12, or the newest
+        version this provider supports below it (so switching clouds just works)."""
+        if not self.runtimes or PREFERRED_RUNTIME in self.runtimes:
+            return PREFERRED_RUNTIME
+
+        def minor(rt: str) -> int:
+            return int(rt.split(".")[1])
+
+        older = [rt for rt in self.runtimes if minor(rt) <= minor(PREFERRED_RUNTIME)]
+        return max(older or list(self.runtimes), key=minor)
+
+    def runtime_id(self, runtime: str) -> str:
+        if self.runtimes and runtime not in self.runtimes:
+            supported = ", ".join(sorted(self.runtimes))
+            raise ProviderError(f"{self.display_name} does not support {runtime}; use one of: {supported}")
+        return self.runtimes.get(runtime, runtime)
+
+    def region(self, config: ServiceConfig) -> str:
+        return config.provider.region or self.default_region
+
+    def pip_platform(self, runtime: str, architecture: str) -> List[str]:
+        """pip flags that download Linux wheels matching the target runtime."""
+        version = runtime.replace("python", "")
+        machine = "aarch64" if architecture == "arm64" else "x86_64"
+        return [
+            "--platform",
+            f"manylinux2014_{machine}",
+            "--implementation",
+            "cp",
+            "--python-version",
+            version,
+            "--only-binary=:all:",
+        ]
+
+    # ------------------------------------------------------------ validation
+    def workflow_engine(self, wf: WorkflowConfig) -> str:
+        if wf.engine == "native":
+            if WORKFLOW_NATIVE not in self.capabilities:
+                raise ProviderError(
+                    f"{self.display_name} has no native workflow service supported by pskit; "
+                    "use `engine: embedded` or `auto`"
+                )
+            return "native"
+        if wf.engine == "embedded":
+            return "embedded"
+        return "native" if WORKFLOW_NATIVE in self.capabilities else "embedded"
+
+    def validate(self, config: ServiceConfig) -> List[Issue]:
+        issues: List[Issue] = []
+        try:
+            self.runtime_id(config.provider.runtime)
+        except ProviderError as exc:
+            issues.append(Issue("error", str(exc), "provider.runtime"))
+        for name, fn in config.functions.items():
+            where = f"functions.{name}"
+            if fn.runtime:
+                try:
+                    self.runtime_id(fn.runtime)
+                except ProviderError as exc:
+                    issues.append(Issue("error", str(exc), where + ".runtime"))
+            for ev in fn.events:
+                if ev.type not in self.capabilities:
+                    issues.append(
+                        Issue(
+                            "error",
+                            f"{self.display_name} does not support `{ev.type}` events in pskit "
+                            f"(supported: {', '.join(sorted(self.capabilities & EVENT_CAPABILITIES))})",
+                            where,
+                        )
+                    )
+            if config.timeout_of(fn) > self.max_timeout:
+                issues.append(
+                    Issue(
+                        "error",
+                        f"timeout {config.timeout_of(fn)}s exceeds {self.display_name} max {self.max_timeout}s",
+                        where,
+                    )
+                )
+        for name, wf in config.workflows.items():
+            try:
+                engine = self.workflow_engine(wf)
+            except ProviderError as exc:
+                issues.append(Issue("error", str(exc), f"workflows.{name}"))
+                continue
+            if engine == "embedded":
+                issues.append(
+                    Issue(
+                        "warning",
+                        "runs on the embedded engine inside one function: total runtime is capped by the "
+                        f"function timeout ({self.max_timeout}s max on {self.display_name})",
+                        f"workflows.{name}",
+                    )
+                )
+        issues.extend(self.extra_validation(config))
+        return issues
+
+    def extra_validation(self, config: ServiceConfig) -> List[Issue]:
+        return []
+
+    # ------------------------------------------------------------ terraform
+    def init_doc(self, ctx: RenderContext) -> TerraformDoc:
+        doc = TerraformDoc()
+        tf = ctx.config.provider.terraform
+        doc.required_version(tf.required_version)
+        local = self.terraform_local_name or self.name
+        doc.required_provider(local, self.terraform_source, tf.provider_version or self.terraform_version)
+        state = ctx.config.provider.state
+        if state.backend:
+            doc.backend(state.backend, state.config)
+        block = self.provider_block(ctx)
+        if block is not None:
+            doc.provider(local, block)
+        return doc
+
+    def provider_block(self, ctx: RenderContext) -> Optional[Dict[str, Any]]:
+        return {"region": ctx.region} if ctx.region else {}
+
+    def render(self, ctx: RenderContext) -> TerraformDoc:
+        doc = self.init_doc(ctx)
+        self.render_resources(doc, ctx)
+        extra = ctx.config.provider.terraform.extra
+        if extra:
+            doc.merge(extra)
+        return doc
+
+    @abstractmethod
+    def render_resources(self, doc: TerraformDoc, ctx: RenderContext) -> None:
+        """Add every resource for functions, triggers and workflows to ``doc``.
+
+        Must emit an output named ``pskit`` shaped like::
+
+            {"functions": {"<name>": {"id": ..., "url": ...}}, "workflows": {...}, "endpoint": ...}
+        """
+
+    # ------------------------------------------------------------ bundle
+    @abstractmethod
+    def shim_files(self, config: ServiceConfig) -> Dict[str, str]:
+        """Files written at the bundle root that forward to the adapter."""
+
+    def bundle_requirements(self, user_requirements: List[str]) -> List[str]:
+        return list(user_requirements) + [r for r in self.extra_requirements if r not in user_requirements]
+
+    def describe(self) -> Dict[str, Any]:
+        return {
+            "name": self.name,
+            "display_name": self.display_name,
+            "tier": self.tier,
+            "default_region": self.default_region,
+            "capabilities": sorted(self.capabilities),
+            "runtimes": sorted(self.runtimes),
+            "max_timeout": self.max_timeout,
+            "terraform_provider": self.terraform_source,
+            "notes": self.notes,
+        }

py_serverless_kit/providers/cloudflare.py

@@ -0,0 +1,264 @@
+"""Cloudflare Python Workers (Pyodide), one Worker script per function.
+
+Each Worker is uploaded as a single Python module. pskit generates that
+module (``.pskit/cloudflare-<stage>/cloudflare/<fn>.py``) by embedding every
+``.py`` file of the bundle - your code plus the pskit runtime - and installing
+an import hook that serves them from memory. That keeps your project layout
+(packages, relative imports) working without a multi-module upload.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..config.schema import ServiceConfig
+from . import schedule as sched
+from .base import HTTP, SCHEDULE, Issue, Provider, RenderContext
+from .terraform_doc import TerraformDoc, ref
+
+SCRIPT = "cloudflare_workers_script"
+WORKER_DIR = "cloudflare"
+DEFAULT_COMPAT_DATE = "2025-09-01"
+
+_LOADER = '''\
+import importlib.abc
+import importlib.util
+import json
+import sys
+
+_PSKIT_ROOT = "/pskit_bundle"
+
+
+class _PskitImporter(importlib.abc.MetaPathFinder, importlib.abc.Loader):
+    """Serves the embedded bundle modules from memory."""
+
+    def __init__(self, files):
+        self._modules = {}
+        for path, source in files.items():
+            if not path.endswith(".py"):
+                continue
+            name = path[:-3].replace("/", ".")
+            is_pkg = name == "__init__" or name.endswith(".__init__")
+            if is_pkg:
+                name = name[: -len(".__init__")] if "." in name else ""
+            if name:
+                self._modules[name] = (path, source, is_pkg)
+        for name in list(self._modules):
+            parts = name.split(".")
+            for i in range(1, len(parts)):
+                parent = ".".join(parts[:i])
+                if parent not in self._modules:  # implicit namespace package
+                    self._modules[parent] = ("/".join(parts[:i]) + "/__init__.py", "", True)
+
+    def find_spec(self, fullname, path=None, target=None):
+        entry = self._modules.get(fullname)
+        if entry is None:
+            return None
+        return importlib.util.spec_from_loader(
+            fullname, self, origin=_PSKIT_ROOT + "/" + entry[0], is_package=entry[2]
+        )
+
+    def create_module(self, spec):
+        return None
+
+    def exec_module(self, module):
+        path, source, is_pkg = self._modules[module.__spec__.name]
+        module.__file__ = _PSKIT_ROOT + "/" + path
+        if is_pkg:
+            module.__path__ = [_PSKIT_ROOT + "/" + path.rsplit("/", 1)[0]]
+        exec(compile(source, module.__file__, "exec"), module.__dict__)
+
+
+sys.meta_path.insert(0, _PskitImporter(_PSKIT_FILES))
+
+from py_serverless_kit.runtime import manifest as _pskit_manifest  # noqa: E402
+from py_serverless_kit.runtime.adapters import cloudflare as _pskit_cf  # noqa: E402
+
+_pskit_manifest.set_manifest(
+    _pskit_manifest.BundleManifest.from_dict(json.loads(_PSKIT_FILES["pskit_manifest.json"]))
+)
+_pskit_cf.configure(_PSKIT_FUNCTION, _PSKIT_ENV, _PSKIT_SCHEDULES)
+
+from workers import WorkerEntrypoint  # noqa: E402
+
+
+class Default(WorkerEntrypoint):
+    async def fetch(self, request):
+        return await _pskit_cf.fetch(request, getattr(self, "env", None))
+
+    async def scheduled(self, controller, env=None, ctx=None):
+        return await _pskit_cf.scheduled(controller, env if env is not None else getattr(self, "env", None))
+'''
+
+
+def embedded_files(build_dir: Path) -> tuple[Dict[str, str], List[str]]:
+    """Collect ``.py`` files and the manifest from the bundle; report skipped files."""
+    files: Dict[str, str] = {}
+    skipped: List[str] = []
+    for path in sorted(p for p in build_dir.rglob("*") if p.is_file()):
+        rel = path.relative_to(build_dir).as_posix()
+        if "__pycache__" in rel or rel == "requirements.txt":
+            continue
+        if rel.endswith(".py") or rel == "pskit_manifest.json":
+            files[rel] = path.read_text("utf-8")
+        else:
+            skipped.append(rel)
+    return files, skipped
+
+
+def worker_source(files: Dict[str, str], function: str, env: Dict[str, str], schedules: Dict[str, Any]) -> str:
+    lines = [
+        "# Generated by py-serverless-kit - do not edit. Cloudflare Python Worker entry module.",
+        f"_PSKIT_FUNCTION = {function!r}",
+        f"_PSKIT_ENV = {dict(sorted(env.items()))!r}",
+        f"_PSKIT_SCHEDULES = {json.loads(json.dumps(schedules))!r}",
+        "_PSKIT_FILES = {",
+    ]
+    for rel, source in files.items():
+        lines.append(f"    {rel!r}: {source!r},")
+    lines.append("}")
+    return "\n".join(lines) + "\n\n" + _LOADER
+
+
+class CloudflareProvider(Provider):
+    name = "cloudflare"
+    display_name = "Cloudflare Workers (Python)"
+    tier = "experimental"
+    terraform_source = "cloudflare/cloudflare"
+    terraform_version = ">= 5.28, < 6.0"
+    default_region = ""
+    capabilities = frozenset({HTTP, SCHEDULE})
+    max_name_length = 63
+    name_pattern = r"[^a-z0-9_-]"
+    runtimes = {f"python3.{v}": f"python3.{v}" for v in (11, 12, 13)}
+    vendor_dependencies = False
+    max_timeout = 900
+    notes = (
+        "Python Workers run on Pyodide: only the standard library and Pyodide built-in packages are available "
+        "(requirements.txt is NOT bundled). One Worker per function; http via workers.dev (set "
+        "options.workers_subdomain for URLs) or options.zone_id + functions.<fn>.options.route; schedules via "
+        "Cron Triggers. Needs options.account_id and CLOUDFLARE_API_TOKEN."
+    )
+
+    # ------------------------------------------------------------------ bundle
+    def shim_files(self, config: ServiceConfig) -> Dict[str, str]:
+        return {}  # the entry module is generated per function in render_resources
+
+    def provider_block(self, ctx: RenderContext) -> Dict[str, Any]:
+        return {}
+
+    def region(self, config: ServiceConfig) -> str:
+        return config.provider.region or ""
+
+    def extra_validation(self, config: ServiceConfig) -> List[Issue]:
+        issues: List[Issue] = []
+        if not config.provider.options.get("account_id"):
+            issues.append(Issue("error", "options.account_id is required", "provider.options.account_id"))
+        issues.append(
+            Issue(
+                "warning",
+                "Cloudflare Python Workers only provide the standard library and Pyodide built-in packages; "
+                "dependencies from requirements.txt are not bundled",
+                "package",
+            )
+        )
+        for name, fn in config.functions.items():
+            for ev in fn.events_of("http"):
+                if ev.auth != "none":
+                    issues.append(
+                        Issue(
+                            "error",
+                            f"auth `{ev.auth}` is not supported on Workers; validate tokens "
+                            "in the handler or use Cloudflare Access",
+                            f"functions.{name}",
+                        )
+                    )
+            if fn.options.get("route") and not config.provider.options.get("zone_id"):
+                issues.append(Issue("error", "options.route needs provider.options.zone_id", f"functions.{name}"))
+        return issues
+
+    # ------------------------------------------------------------------ render
+    def render_resources(self, doc: TerraformDoc, ctx: RenderContext) -> None:
+        cfg = ctx.config
+        account = str(ctx.opt("account_id", ""))
+        subdomain = ctx.opt("workers_subdomain")
+        workers_dev = bool(ctx.opt("workers_dev", True))
+        build = ctx.artifact.build_dir
+        files, skipped = embedded_files(Path(build)) if build is not None else ({}, [])
+        if skipped:
+            doc.warnings.append(
+                "Cloudflare: non-Python files are not uploaded: "
+                + ", ".join(skipped[:10])
+                + (" ..." if len(skipped) > 10 else "")
+            )
+        out_dir = ctx.work_dir / WORKER_DIR
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+        fn_out: Dict[str, Dict[str, Any]] = {}
+        endpoint: Optional[str] = None
+        for name, fn in ctx.function_items():
+            script_name = ctx.physical_name(name)
+            schedules = {sched.to_cron5(ev): ev.input for ev in fn.events_of("schedule") if ev.enabled}
+            source = worker_source(files, name, ctx.environment(name, fn), schedules)
+            path = out_dir / f"{name}.py"
+            path.write_text(source, "utf-8", newline="\n")
+            opts = dict(fn.options)
+            route = opts.pop("route", None)
+            body: Dict[str, Any] = {
+                "account_id": account,
+                "script_name": script_name,
+                "content_file": f"${{path.module}}/{WORKER_DIR}/{name}.py",
+                "content_sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(),
+                "content_type": "text/x-python",
+                "main_module": "worker.py",
+                "compatibility_date": str(ctx.opt("compatibility_date", DEFAULT_COMPAT_DATE)),
+                "compatibility_flags": ["python_workers"],
+            }
+            body.update(opts)
+            script = doc.resource(SCRIPT, name, body)
+            http = bool(fn.events_of("http"))
+            if http and workers_dev:
+                doc.resource(
+                    "cloudflare_workers_script_subdomain",
+                    name,
+                    {"account_id": account, "script_name": ref(script, "script_name"), "enabled": True},
+                )
+            if route:
+                doc.resource(
+                    "cloudflare_workers_route",
+                    name,
+                    {"zone_id": str(ctx.opt("zone_id", "")), "pattern": route, "script": ref(script, "script_name")},
+                )
+            if schedules:
+                doc.resource(
+                    "cloudflare_workers_cron_trigger",
+                    name,
+                    {
+                        "account_id": account,
+                        "script_name": ref(script, "script_name"),
+                        "schedules": [{"cron": c} for c in schedules],
+                    },
+                )
+            out: Dict[str, Any] = {
+                "id": ref(script, "script_name"),
+                "console": f"https://dash.cloudflare.com/{account}/workers/services/view/{script_name}/production",
+            }
+            url = None
+            if http and workers_dev and subdomain:
+                url = f"https://{script_name}.{subdomain}.workers.dev"
+            elif http and route:
+                url = "https://" + str(route).rstrip("*").rstrip("/")
+            if url:
+                out["url"] = url
+                out["base_url"] = url
+                endpoint = endpoint or url
+            fn_out[name] = out
+        wf_out = {n: {"id": ctx.physical_name(f"{n}-orchestrator")} for n in cfg.workflows}
+        doc.output(
+            "pskit",
+            {"functions": fn_out, "workflows": wf_out, "endpoint": endpoint},
+            description="py-serverless-kit deployment summary",
+        )