py-serverless-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_serverless_kit/__init__.py +47 -0
- py_serverless_kit/__main__.py +3 -0
- py_serverless_kit/_version.py +2 -0
- py_serverless_kit/cli/__init__.py +0 -0
- py_serverless_kit/cli/main.py +491 -0
- py_serverless_kit/cli/templates.py +212 -0
- py_serverless_kit/config/__init__.py +12 -0
- py_serverless_kit/config/loader.py +316 -0
- py_serverless_kit/config/schema.py +457 -0
- py_serverless_kit/core/__init__.py +0 -0
- py_serverless_kit/core/deployer.py +180 -0
- py_serverless_kit/core/describe.py +112 -0
- py_serverless_kit/core/project.py +147 -0
- py_serverless_kit/dashboard/__init__.py +3 -0
- py_serverless_kit/dashboard/generator.py +65 -0
- py_serverless_kit/dashboard/template.html +248 -0
- py_serverless_kit/local/__init__.py +3 -0
- py_serverless_kit/local/runner.py +234 -0
- py_serverless_kit/local/server.py +200 -0
- py_serverless_kit/packaging/__init__.py +3 -0
- py_serverless_kit/packaging/builder.py +252 -0
- py_serverless_kit/providers/__init__.py +15 -0
- py_serverless_kit/providers/alibaba.py +268 -0
- py_serverless_kit/providers/aws.py +508 -0
- py_serverless_kit/providers/azure.py +417 -0
- py_serverless_kit/providers/base.py +310 -0
- py_serverless_kit/providers/cloudflare.py +264 -0
- py_serverless_kit/providers/digitalocean.py +211 -0
- py_serverless_kit/providers/gcp.py +480 -0
- py_serverless_kit/providers/huawei.py +292 -0
- py_serverless_kit/providers/ibm.py +132 -0
- py_serverless_kit/providers/oracle.py +304 -0
- py_serverless_kit/providers/registry.py +60 -0
- py_serverless_kit/providers/schedule.py +153 -0
- py_serverless_kit/providers/tencent.py +275 -0
- py_serverless_kit/providers/terraform_doc.py +125 -0
- py_serverless_kit/runtime/__init__.py +6 -0
- py_serverless_kit/runtime/adapters/__init__.py +18 -0
- py_serverless_kit/runtime/adapters/_httpfn.py +91 -0
- py_serverless_kit/runtime/adapters/alibaba.py +147 -0
- py_serverless_kit/runtime/adapters/aws.py +201 -0
- py_serverless_kit/runtime/adapters/azure.py +185 -0
- py_serverless_kit/runtime/adapters/base.py +86 -0
- py_serverless_kit/runtime/adapters/cloudflare.py +126 -0
- py_serverless_kit/runtime/adapters/digitalocean.py +74 -0
- py_serverless_kit/runtime/adapters/gcp.py +176 -0
- py_serverless_kit/runtime/adapters/huawei.py +173 -0
- py_serverless_kit/runtime/adapters/ibm.py +91 -0
- py_serverless_kit/runtime/adapters/oracle.py +161 -0
- py_serverless_kit/runtime/adapters/tencent.py +151 -0
- py_serverless_kit/runtime/batch.py +47 -0
- py_serverless_kit/runtime/context.py +45 -0
- py_serverless_kit/runtime/dispatch.py +277 -0
- py_serverless_kit/runtime/errors.py +83 -0
- py_serverless_kit/runtime/event.py +267 -0
- py_serverless_kit/runtime/invoke.py +119 -0
- py_serverless_kit/runtime/log.py +69 -0
- py_serverless_kit/runtime/manifest.py +121 -0
- py_serverless_kit/runtime/response.py +177 -0
- py_serverless_kit/runtime/routing.py +74 -0
- py_serverless_kit/runtime/workflow.py +312 -0
- py_serverless_kit/terraform/__init__.py +3 -0
- py_serverless_kit/terraform/runner.py +104 -0
- py_serverless_kit/workflows/__init__.py +35 -0
- py_serverless_kit/workflows/asl.py +170 -0
- py_serverless_kit/workflows/gcp.py +328 -0
- py_serverless_kit-0.1.0.dist-info/METADATA +300 -0
- py_serverless_kit-0.1.0.dist-info/RECORD +71 -0
- py_serverless_kit-0.1.0.dist-info/WHEEL +4 -0
- py_serverless_kit-0.1.0.dist-info/entry_points.txt +3 -0
- py_serverless_kit-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Run handlers and workflows on your machine, exactly as the cloud adapters would."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Dict, Iterator, List, Optional
|
|
13
|
+
from urllib.parse import parse_qsl, urlsplit
|
|
14
|
+
|
|
15
|
+
from ..core.project import Project
|
|
16
|
+
from ..runtime import dispatch
|
|
17
|
+
from ..runtime.context import Context
|
|
18
|
+
from ..runtime.event import Event, EventType, Headers, HttpRequest, Record
|
|
19
|
+
from ..runtime.manifest import BundleManifest, set_manifest
|
|
20
|
+
from ..runtime.workflow import WorkflowEngine, in_process_runner
|
|
21
|
+
from ..workflows import definition_dict
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def load_dotenv(path: Path) -> Dict[str, str]:
|
|
25
|
+
"""Minimal ``.env`` parser (KEY=VALUE, comments, optional quotes)."""
|
|
26
|
+
values: Dict[str, str] = {}
|
|
27
|
+
if not path.is_file():
|
|
28
|
+
return values
|
|
29
|
+
for line in path.read_text("utf-8").splitlines():
|
|
30
|
+
line = line.strip()
|
|
31
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
32
|
+
continue
|
|
33
|
+
key, _, value = line.partition("=")
|
|
34
|
+
key = key.strip()
|
|
35
|
+
if key.startswith("export "):
|
|
36
|
+
key = key[len("export ") :].strip()
|
|
37
|
+
value = value.strip()
|
|
38
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
|
|
39
|
+
value = value[1:-1]
|
|
40
|
+
values[key] = value
|
|
41
|
+
return values
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class LocalRunner:
|
|
45
|
+
"""Executes a project's functions in-process with the real dispatcher."""
|
|
46
|
+
|
|
47
|
+
def __init__(self, project: Project, env_file: Optional[str] = ".env") -> None:
|
|
48
|
+
self.project = project
|
|
49
|
+
self.source = (project.root / project.config.package.source).resolve()
|
|
50
|
+
self.env_file = project.root / env_file if env_file else None
|
|
51
|
+
manifest = project.bundle_manifest()
|
|
52
|
+
# Every workflow is runnable locally, whatever engine the cloud uses.
|
|
53
|
+
manifest["workflows"] = {
|
|
54
|
+
name: {"orchestrator": f"{name}-orchestrator", "definition": definition_dict(wf)}
|
|
55
|
+
for name, wf in project.config.workflows.items()
|
|
56
|
+
}
|
|
57
|
+
self.manifest = BundleManifest.from_dict(manifest)
|
|
58
|
+
self._purge_foreign_modules()
|
|
59
|
+
|
|
60
|
+
def _purge_foreign_modules(self) -> None:
|
|
61
|
+
"""Drop cached modules that shadow this project's packages (e.g. another
|
|
62
|
+
project's ``app.handlers`` imported earlier in the same process)."""
|
|
63
|
+
try:
|
|
64
|
+
top_level = {p.stem if p.suffix == ".py" else p.name for p in self.source.iterdir()}
|
|
65
|
+
except OSError:
|
|
66
|
+
return
|
|
67
|
+
src = str(self.source)
|
|
68
|
+
for name, module in list(sys.modules.items()):
|
|
69
|
+
if name.split(".")[0] not in top_level:
|
|
70
|
+
continue
|
|
71
|
+
path = getattr(module, "__file__", None) or ""
|
|
72
|
+
if not path.startswith(src):
|
|
73
|
+
del sys.modules[name]
|
|
74
|
+
dispatch._HANDLERS.clear()
|
|
75
|
+
|
|
76
|
+
# ---------------------------------------------------------------- env
|
|
77
|
+
@contextmanager
|
|
78
|
+
def environment(self, function: Optional[str] = None) -> Iterator[None]:
|
|
79
|
+
cfg = self.project.config
|
|
80
|
+
env: Dict[str, str] = {
|
|
81
|
+
"PSKIT_PROVIDER": "local",
|
|
82
|
+
"PSKIT_SERVICE": cfg.service,
|
|
83
|
+
"PSKIT_STAGE": cfg.provider.stage,
|
|
84
|
+
"PSKIT_REGION": "local",
|
|
85
|
+
"PSKIT_LOG_FORMAT": os.environ.get("PSKIT_LOG_FORMAT", "text"),
|
|
86
|
+
}
|
|
87
|
+
env.update(cfg.provider.environment)
|
|
88
|
+
if function and function in cfg.functions:
|
|
89
|
+
env.update(cfg.functions[function].environment)
|
|
90
|
+
env["PSKIT_FUNCTION"] = function
|
|
91
|
+
env["PSKIT_HANDLER"] = cfg.functions[function].handler
|
|
92
|
+
if self.env_file is not None:
|
|
93
|
+
env.update(load_dotenv(self.env_file))
|
|
94
|
+
saved = {k: os.environ.get(k) for k in env}
|
|
95
|
+
os.environ.update(env)
|
|
96
|
+
added_path = str(self.source)
|
|
97
|
+
sys.path.insert(0, added_path)
|
|
98
|
+
set_manifest(self.manifest)
|
|
99
|
+
try:
|
|
100
|
+
yield
|
|
101
|
+
finally:
|
|
102
|
+
for key, old in saved.items():
|
|
103
|
+
if old is None:
|
|
104
|
+
os.environ.pop(key, None)
|
|
105
|
+
else:
|
|
106
|
+
os.environ[key] = old
|
|
107
|
+
if added_path in sys.path:
|
|
108
|
+
sys.path.remove(added_path)
|
|
109
|
+
|
|
110
|
+
# ---------------------------------------------------------------- invoke
|
|
111
|
+
def invoke(self, function: str, event: Event) -> Any:
|
|
112
|
+
if function not in self.project.config.functions:
|
|
113
|
+
known = ", ".join(sorted(self.project.config.functions))
|
|
114
|
+
raise KeyError(f"unknown function {function!r}; known: {known}")
|
|
115
|
+
event.function = function
|
|
116
|
+
event.provider = "local"
|
|
117
|
+
with self.environment(function):
|
|
118
|
+
return dispatch.execute(event, Context(function=function, provider="local"), function)
|
|
119
|
+
|
|
120
|
+
def run_workflow(self, name: str, data: Any, *, sleep: bool = False) -> Dict[str, Any]:
|
|
121
|
+
wf = self.project.config.workflows.get(name)
|
|
122
|
+
if wf is None:
|
|
123
|
+
known = ", ".join(sorted(self.project.config.workflows)) or "<none>"
|
|
124
|
+
raise KeyError(f"unknown workflow {name!r}; known: {known}")
|
|
125
|
+
engine = WorkflowEngine(
|
|
126
|
+
definition_dict(wf),
|
|
127
|
+
in_process_runner("local"),
|
|
128
|
+
sleep=time.sleep if sleep else (lambda _seconds: None),
|
|
129
|
+
)
|
|
130
|
+
with self.environment():
|
|
131
|
+
output = engine.run(data)
|
|
132
|
+
return {"output": output, "history": engine.history}
|
|
133
|
+
|
|
134
|
+
def reload(self) -> None:
|
|
135
|
+
"""Forget imported user modules so edited code is picked up (dev server)."""
|
|
136
|
+
src = str(self.source)
|
|
137
|
+
for name, module in list(sys.modules.items()):
|
|
138
|
+
path = getattr(module, "__file__", None) or ""
|
|
139
|
+
if path.startswith(src) and "py_serverless_kit" not in path:
|
|
140
|
+
del sys.modules[name]
|
|
141
|
+
dispatch._HANDLERS.clear()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# -------------------------------------------------------------------- event builders
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def http_event(
|
|
148
|
+
method: str = "GET",
|
|
149
|
+
path: str = "/",
|
|
150
|
+
body: Any = None,
|
|
151
|
+
headers: Optional[Dict[str, str]] = None,
|
|
152
|
+
query: Optional[Dict[str, str]] = None,
|
|
153
|
+
) -> Event:
|
|
154
|
+
parts = urlsplit(path)
|
|
155
|
+
q = dict(parse_qsl(parts.query, keep_blank_values=True))
|
|
156
|
+
q.update(query or {})
|
|
157
|
+
hdrs = dict(headers or {})
|
|
158
|
+
if isinstance(body, (dict, list)):
|
|
159
|
+
body = json.dumps(body)
|
|
160
|
+
hdrs.setdefault("content-type", "application/json")
|
|
161
|
+
return Event(
|
|
162
|
+
type=EventType.HTTP,
|
|
163
|
+
http=HttpRequest(
|
|
164
|
+
method=method,
|
|
165
|
+
path=parts.path or "/",
|
|
166
|
+
headers=Headers(hdrs),
|
|
167
|
+
query=q,
|
|
168
|
+
body=body or b"",
|
|
169
|
+
source_ip="127.0.0.1",
|
|
170
|
+
),
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def build_event(kind: str, data: Any = None, **http: Any) -> Event:
|
|
175
|
+
"""Create a synthetic event for ``pskit invoke``."""
|
|
176
|
+
if kind == EventType.HTTP:
|
|
177
|
+
return http_event(
|
|
178
|
+
http.get("method", "GET"), http.get("path", "/"), data, http.get("headers"), http.get("query")
|
|
179
|
+
)
|
|
180
|
+
if kind in (EventType.QUEUE, EventType.TOPIC):
|
|
181
|
+
items: List[Any] = data if isinstance(data, list) else [data]
|
|
182
|
+
records = [
|
|
183
|
+
Record(id=f"local-{i}", body=json.dumps(item) if isinstance(item, (dict, list)) else item, source="local")
|
|
184
|
+
for i, item in enumerate(items)
|
|
185
|
+
]
|
|
186
|
+
return Event(type=kind, records=records, source="local")
|
|
187
|
+
if kind == EventType.STORAGE:
|
|
188
|
+
items = data if isinstance(data, list) else [data or {}]
|
|
189
|
+
records = [
|
|
190
|
+
Record(
|
|
191
|
+
id=f"local-{i}",
|
|
192
|
+
bucket=(item or {}).get("bucket", "local-bucket"),
|
|
193
|
+
key=(item or {}).get("key", "example.txt"),
|
|
194
|
+
size=(item or {}).get("size", 0),
|
|
195
|
+
action=(item or {}).get("action", "created"),
|
|
196
|
+
source="local",
|
|
197
|
+
)
|
|
198
|
+
for i, item in enumerate(items)
|
|
199
|
+
]
|
|
200
|
+
return Event(type=kind, records=records, source="local")
|
|
201
|
+
if kind not in EventType.ALL:
|
|
202
|
+
raise ValueError(f"unknown event type {kind!r}; one of {', '.join(EventType.ALL)}")
|
|
203
|
+
return Event(type=kind, data=data)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def parse_data(value: Optional[str]) -> Any:
|
|
207
|
+
"""``--data '{"a":1}'`` or ``--data @event.json`` (or plain text)."""
|
|
208
|
+
if value is None:
|
|
209
|
+
return None
|
|
210
|
+
if value.startswith("@"):
|
|
211
|
+
value = Path(value[1:]).read_text("utf-8")
|
|
212
|
+
try:
|
|
213
|
+
return json.loads(value)
|
|
214
|
+
except ValueError:
|
|
215
|
+
return value
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def response_to_dict(result: Any) -> Any:
|
|
219
|
+
from ..runtime.batch import BatchResult
|
|
220
|
+
from ..runtime.response import Response
|
|
221
|
+
|
|
222
|
+
if isinstance(result, BatchResult):
|
|
223
|
+
return {"batchItemFailures": [{"itemIdentifier": rid} for rid in result.failed]}
|
|
224
|
+
if isinstance(result, Response):
|
|
225
|
+
status, headers, raw = result.render()
|
|
226
|
+
try:
|
|
227
|
+
body: Any = json.loads(raw) if raw else None
|
|
228
|
+
except ValueError:
|
|
229
|
+
try:
|
|
230
|
+
body = raw.decode("utf-8")
|
|
231
|
+
except UnicodeDecodeError:
|
|
232
|
+
body = {"base64": base64.b64encode(raw).decode("ascii")}
|
|
233
|
+
return {"status": status, "headers": headers, "body": body}
|
|
234
|
+
return result
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""``pskit dev`` - a local HTTP server that routes requests to your handlers.
|
|
2
|
+
|
|
3
|
+
* Every ``http`` event in the config becomes a route (``{id}`` / ``{path+}`` supported).
|
|
4
|
+
* ``POST /__pskit/invoke/<function>`` invoke any function with a JSON body
|
|
5
|
+
(``?type=queue|topic|storage|schedule|invoke``; default ``invoke``).
|
|
6
|
+
* ``POST /__pskit/workflows/<workflow>`` run a workflow on the embedded engine.
|
|
7
|
+
* ``GET /__pskit/`` the dashboard (local view of the project).
|
|
8
|
+
* Code changes are picked up on the next request (no restart needed).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import threading
|
|
16
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
19
|
+
from urllib.parse import parse_qsl, urlsplit
|
|
20
|
+
|
|
21
|
+
from ..runtime.event import Event, EventType, Headers, HttpRequest
|
|
22
|
+
from ..runtime.response import Response, dumps
|
|
23
|
+
from ..runtime.routing import match_path, method_matches
|
|
24
|
+
from ..runtime.workflow import WorkflowFailed
|
|
25
|
+
from .runner import LocalRunner, build_event
|
|
26
|
+
|
|
27
|
+
Route = Tuple[str, str, str] # (method, path, function)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _routes(runner: LocalRunner) -> List[Route]:
|
|
31
|
+
routes = []
|
|
32
|
+
for name, fn in runner.project.config.functions.items():
|
|
33
|
+
for ev in fn.events_of("http"):
|
|
34
|
+
routes.append((ev.method, ev.path, name))
|
|
35
|
+
# Static segments first so /users/me wins over /users/{id}.
|
|
36
|
+
routes.sort(key=lambda r: (r[1].count("{"), -len(r[1])))
|
|
37
|
+
return routes
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class _Watcher:
|
|
41
|
+
"""Detects source changes by polling mtimes on request (cheap and portable)."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, root: Path) -> None:
|
|
44
|
+
self.root = root
|
|
45
|
+
self.stamp = self._scan()
|
|
46
|
+
self.lock = threading.Lock()
|
|
47
|
+
|
|
48
|
+
def _scan(self) -> float:
|
|
49
|
+
latest = 0.0
|
|
50
|
+
for dirpath, dirnames, filenames in os.walk(self.root):
|
|
51
|
+
dirnames[:] = [
|
|
52
|
+
d for d in dirnames if not d.startswith(".") and d not in ("__pycache__", "venv", "node_modules")
|
|
53
|
+
]
|
|
54
|
+
for f in filenames:
|
|
55
|
+
if f.endswith(".py"):
|
|
56
|
+
try:
|
|
57
|
+
latest = max(latest, os.stat(os.path.join(dirpath, f)).st_mtime)
|
|
58
|
+
except OSError:
|
|
59
|
+
pass
|
|
60
|
+
return latest
|
|
61
|
+
|
|
62
|
+
def changed(self) -> bool:
|
|
63
|
+
with self.lock:
|
|
64
|
+
now = self._scan()
|
|
65
|
+
if now != self.stamp:
|
|
66
|
+
self.stamp = now
|
|
67
|
+
return True
|
|
68
|
+
return False
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def make_handler(runner: LocalRunner, echo=print): # noqa: ANN001
|
|
72
|
+
routes = _routes(runner)
|
|
73
|
+
watcher = _Watcher(runner.source)
|
|
74
|
+
lock = threading.Lock() # handlers share os.environ; serialize invocations
|
|
75
|
+
|
|
76
|
+
class Handler(BaseHTTPRequestHandler):
|
|
77
|
+
server_version = "pskit-dev"
|
|
78
|
+
|
|
79
|
+
def log_message(self, fmt: str, *args: Any) -> None: # quieter default logging
|
|
80
|
+
pass
|
|
81
|
+
|
|
82
|
+
def _send(self, status: int, headers: Dict[str, str], body: bytes, cookies: Optional[List[str]] = None) -> None:
|
|
83
|
+
self.send_response(status)
|
|
84
|
+
for k, v in headers.items():
|
|
85
|
+
self.send_header(k, v)
|
|
86
|
+
for cookie in cookies or []:
|
|
87
|
+
self.send_header("set-cookie", cookie)
|
|
88
|
+
self.send_header("content-length", str(len(body)))
|
|
89
|
+
self.end_headers()
|
|
90
|
+
if self.command != "HEAD":
|
|
91
|
+
self.wfile.write(body)
|
|
92
|
+
|
|
93
|
+
def _json(self, status: int, payload: Any) -> None:
|
|
94
|
+
self._send(status, {"content-type": "application/json"}, dumps(payload).encode("utf-8"))
|
|
95
|
+
|
|
96
|
+
def _body(self) -> bytes:
|
|
97
|
+
length = int(self.headers.get("content-length") or 0)
|
|
98
|
+
return self.rfile.read(length) if length else b""
|
|
99
|
+
|
|
100
|
+
def _dispatch(self) -> None:
|
|
101
|
+
if watcher.changed():
|
|
102
|
+
runner.reload()
|
|
103
|
+
echo(" (reloaded changed source files)")
|
|
104
|
+
url = urlsplit(self.path)
|
|
105
|
+
if url.path.startswith("/__pskit"):
|
|
106
|
+
return self._internal(url.path, dict(parse_qsl(url.query)))
|
|
107
|
+
for method, pattern, fn in routes:
|
|
108
|
+
params = match_path(pattern, url.path, tolerant=False)
|
|
109
|
+
if params is None or not (method_matches(method, self.command) or self.command == "OPTIONS"):
|
|
110
|
+
continue
|
|
111
|
+
req = HttpRequest(
|
|
112
|
+
method=self.command,
|
|
113
|
+
path=url.path,
|
|
114
|
+
headers=Headers(list(self.headers.items())),
|
|
115
|
+
query=dict(parse_qsl(url.query, keep_blank_values=True)),
|
|
116
|
+
path_params=params,
|
|
117
|
+
body=self._body(),
|
|
118
|
+
source_ip=self.client_address[0],
|
|
119
|
+
)
|
|
120
|
+
with lock:
|
|
121
|
+
resp: Response = runner.invoke(fn, Event(type=EventType.HTTP, http=req))
|
|
122
|
+
status, headers, raw = resp.render()
|
|
123
|
+
echo(f" {self.command} {url.path} -> {fn} [{status}]")
|
|
124
|
+
return self._send(status, headers, raw, resp.cookies)
|
|
125
|
+
self._json(404, {"error": "Not Found", "status": 404, "routes": [f"{m} {p}" for m, p, _ in routes]})
|
|
126
|
+
|
|
127
|
+
def _internal(self, path: str, query: Dict[str, str]) -> None:
|
|
128
|
+
parts = [p for p in path.split("/") if p][1:]
|
|
129
|
+
if not parts or parts == ["dashboard"]:
|
|
130
|
+
from ..dashboard.generator import local_record, render_html
|
|
131
|
+
|
|
132
|
+
html = render_html([local_record(runner.project)], title=f"{runner.project.config.service} (local)")
|
|
133
|
+
return self._send(200, {"content-type": "text/html; charset=utf-8"}, html.encode("utf-8"))
|
|
134
|
+
raw = self._body()
|
|
135
|
+
try:
|
|
136
|
+
data = json.loads(raw) if raw else None
|
|
137
|
+
except ValueError:
|
|
138
|
+
data = raw.decode("utf-8", "replace")
|
|
139
|
+
if parts[0] == "invoke" and len(parts) == 2:
|
|
140
|
+
kind = query.get("type", EventType.INVOKE)
|
|
141
|
+
try:
|
|
142
|
+
with lock:
|
|
143
|
+
result = runner.invoke(parts[1], build_event(kind, data))
|
|
144
|
+
except KeyError as exc:
|
|
145
|
+
return self._json(404, {"error": str(exc)})
|
|
146
|
+
except Exception as exc: # noqa: BLE001 - surface handler errors to the caller
|
|
147
|
+
return self._json(500, {"errorType": type(exc).__name__, "cause": str(exc)})
|
|
148
|
+
from .runner import response_to_dict
|
|
149
|
+
|
|
150
|
+
echo(f" invoke {parts[1]} ({kind})")
|
|
151
|
+
return self._json(200, {"result": response_to_dict(result)})
|
|
152
|
+
if parts[0] == "workflows" and len(parts) == 2:
|
|
153
|
+
try:
|
|
154
|
+
with lock:
|
|
155
|
+
out = runner.run_workflow(parts[1], data)
|
|
156
|
+
except KeyError as exc:
|
|
157
|
+
return self._json(404, {"error": str(exc)})
|
|
158
|
+
except WorkflowFailed as exc:
|
|
159
|
+
return self._json(
|
|
160
|
+
500, {"status": "FAILED", "error": exc.error, "cause": exc.cause, "history": exc.history}
|
|
161
|
+
)
|
|
162
|
+
echo(f" workflow {parts[1]} SUCCEEDED")
|
|
163
|
+
return self._json(200, {"status": "SUCCEEDED", **out})
|
|
164
|
+
self._json(404, {"error": f"unknown internal path {path}"})
|
|
165
|
+
|
|
166
|
+
do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_HEAD = do_OPTIONS = _dispatch
|
|
167
|
+
|
|
168
|
+
return Handler, routes
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def serve(runner: LocalRunner, host: str = "127.0.0.1", port: int = 3000, echo=print) -> None: # noqa: ANN001
|
|
172
|
+
handler, routes = make_handler(runner, echo)
|
|
173
|
+
httpd = ThreadingHTTPServer((host, port), handler)
|
|
174
|
+
base = f"http://{host}:{port}"
|
|
175
|
+
echo(f"pskit dev server for '{runner.project.config.service}' on {base}")
|
|
176
|
+
for method, path, fn in routes:
|
|
177
|
+
echo(f" {method:7} {base}{path} -> {fn}")
|
|
178
|
+
echo(f" POST {base}/__pskit/invoke/<function>?type=invoke|queue|topic|storage|schedule")
|
|
179
|
+
if runner.project.config.workflows:
|
|
180
|
+
echo(f" POST {base}/__pskit/workflows/<workflow>")
|
|
181
|
+
echo(f" GET {base}/__pskit/ (dashboard)")
|
|
182
|
+
echo("Press Ctrl+C to stop.")
|
|
183
|
+
try:
|
|
184
|
+
httpd.serve_forever()
|
|
185
|
+
except KeyboardInterrupt:
|
|
186
|
+
echo("stopping")
|
|
187
|
+
finally:
|
|
188
|
+
httpd.server_close()
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def start_background(runner: LocalRunner, host: str = "127.0.0.1", port: int = 0) -> Tuple[ThreadingHTTPServer, str]:
|
|
192
|
+
"""Start the server on a thread (used by tests). Returns ``(server, base_url)``."""
|
|
193
|
+
handler, _ = make_handler(runner, echo=lambda _m: None)
|
|
194
|
+
httpd = ThreadingHTTPServer((host, port), handler)
|
|
195
|
+
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
|
196
|
+
thread.start()
|
|
197
|
+
return httpd, f"http://{host}:{httpd.server_address[1]}"
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
__all__ = ["serve", "start_background"]
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"""Build the deployment artifact (a deterministic zip).
|
|
2
|
+
|
|
3
|
+
Layout of the bundle root::
|
|
4
|
+
|
|
5
|
+
<your sources> filtered by package.include / package.exclude
|
|
6
|
+
py_serverless_kit/ the stdlib-only runtime (vendored)
|
|
7
|
+
pskit_manifest.json functions, routes, embedded workflows
|
|
8
|
+
<provider shim files> e.g. pskit_entry.py, main.py, function_app.py
|
|
9
|
+
<dependencies> pip-installed when the provider runs no remote build
|
|
10
|
+
requirements.txt when the provider builds remotely (GCP, Azure)
|
|
11
|
+
|
|
12
|
+
The zip is byte-for-byte reproducible (sorted entries, fixed timestamps), so an
|
|
13
|
+
unchanged project never triggers a redeploy.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import base64
|
|
19
|
+
import hashlib
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import shutil
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
import zipfile
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import TYPE_CHECKING, Iterable, List, Optional, Pattern
|
|
29
|
+
|
|
30
|
+
from ..providers.base import Artifact
|
|
31
|
+
|
|
32
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
33
|
+
from ..core.project import Project
|
|
34
|
+
|
|
35
|
+
DEFAULT_EXCLUDES = (
|
|
36
|
+
".git/**",
|
|
37
|
+
".hg/**",
|
|
38
|
+
".pskit/**",
|
|
39
|
+
".terraform/**",
|
|
40
|
+
".venv/**",
|
|
41
|
+
"venv/**",
|
|
42
|
+
"env/**",
|
|
43
|
+
"**/__pycache__/**",
|
|
44
|
+
"**/*.pyc",
|
|
45
|
+
"**/*.pyo",
|
|
46
|
+
".pytest_cache/**",
|
|
47
|
+
".mypy_cache/**",
|
|
48
|
+
".ruff_cache/**",
|
|
49
|
+
".idea/**",
|
|
50
|
+
".vscode/**",
|
|
51
|
+
"node_modules/**",
|
|
52
|
+
"tests/**",
|
|
53
|
+
"dist/**",
|
|
54
|
+
"build/**",
|
|
55
|
+
"*.egg-info/**",
|
|
56
|
+
"**/.DS_Store",
|
|
57
|
+
".env",
|
|
58
|
+
".env.*",
|
|
59
|
+
"**/*.tfstate",
|
|
60
|
+
"**/*.tfstate.*",
|
|
61
|
+
"pskit*.yml",
|
|
62
|
+
"pskit*.yaml",
|
|
63
|
+
"serverless_kit*.yml",
|
|
64
|
+
)
|
|
65
|
+
_FIXED_DATE = (1980, 1, 1, 0, 0, 0)
|
|
66
|
+
_RUNTIME_PKG = "py_serverless_kit"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class BuildError(Exception):
|
|
70
|
+
pass
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def glob_to_regex(pattern: str) -> Pattern[str]:
|
|
74
|
+
"""Translate a gitignore-like glob (``**`` = any depth) into a regex."""
|
|
75
|
+
pattern = pattern.replace("\\", "/")
|
|
76
|
+
if pattern.startswith("./"):
|
|
77
|
+
pattern = pattern[2:]
|
|
78
|
+
if pattern.endswith("/"):
|
|
79
|
+
pattern += "**"
|
|
80
|
+
i, out = 0, []
|
|
81
|
+
while i < len(pattern):
|
|
82
|
+
c = pattern[i]
|
|
83
|
+
if c == "*":
|
|
84
|
+
if pattern[i : i + 3] == "**/":
|
|
85
|
+
out.append("(?:.*/)?")
|
|
86
|
+
i += 3
|
|
87
|
+
continue
|
|
88
|
+
if pattern[i : i + 2] == "**":
|
|
89
|
+
out.append(".*")
|
|
90
|
+
i += 2
|
|
91
|
+
continue
|
|
92
|
+
out.append("[^/]*")
|
|
93
|
+
elif c == "?":
|
|
94
|
+
out.append("[^/]")
|
|
95
|
+
else:
|
|
96
|
+
out.append(re.escape(c))
|
|
97
|
+
i += 1
|
|
98
|
+
return re.compile("^" + "".join(out) + "$")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _matches(rel: str, patterns: Iterable[Pattern[str]]) -> bool:
|
|
102
|
+
return any(p.match(rel) for p in patterns)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def collect_sources(source: Path, include: List[str], exclude: List[str]) -> List[Path]:
|
|
106
|
+
inc = [glob_to_regex(p) for p in include]
|
|
107
|
+
exc = [glob_to_regex(p) for p in list(DEFAULT_EXCLUDES) + list(exclude)]
|
|
108
|
+
files: List[Path] = []
|
|
109
|
+
for dirpath, dirnames, filenames in os.walk(source):
|
|
110
|
+
rel_dir = Path(dirpath).relative_to(source).as_posix()
|
|
111
|
+
rel_dir = "" if rel_dir == "." else rel_dir + "/"
|
|
112
|
+
# prune excluded directories early (fast on big venvs / node_modules)
|
|
113
|
+
dirnames[:] = sorted(d for d in dirnames if not _matches(f"{rel_dir}{d}/x", exc))
|
|
114
|
+
for name in sorted(filenames):
|
|
115
|
+
rel = f"{rel_dir}{name}"
|
|
116
|
+
if _matches(rel, inc) and not _matches(rel, exc):
|
|
117
|
+
files.append(Path(dirpath) / name)
|
|
118
|
+
return files
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _copy_runtime(dest: Path) -> None:
|
|
122
|
+
pkg_root = Path(__file__).resolve().parents[1]
|
|
123
|
+
target = dest / _RUNTIME_PKG
|
|
124
|
+
if target.exists():
|
|
125
|
+
shutil.rmtree(target)
|
|
126
|
+
target.mkdir(parents=True)
|
|
127
|
+
for name in ("__init__.py", "_version.py"):
|
|
128
|
+
shutil.copy2(pkg_root / name, target / name)
|
|
129
|
+
shutil.copytree(
|
|
130
|
+
pkg_root / "runtime",
|
|
131
|
+
target / "runtime",
|
|
132
|
+
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
|
133
|
+
)
|
|
134
|
+
(target / "py.typed").write_text("", "utf-8")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def read_requirements(path: Optional[Path]) -> List[str]:
|
|
138
|
+
if path is None or not path.is_file():
|
|
139
|
+
return []
|
|
140
|
+
reqs = []
|
|
141
|
+
for line in path.read_text("utf-8").splitlines():
|
|
142
|
+
line = line.strip()
|
|
143
|
+
if not line or line.startswith("#"):
|
|
144
|
+
continue
|
|
145
|
+
# The runtime is vendored; never pull the full CLI (and its deps) into a function.
|
|
146
|
+
if re.match(r"^py[-_]serverless[-_]kit\b", line, re.I):
|
|
147
|
+
continue
|
|
148
|
+
reqs.append(line)
|
|
149
|
+
return reqs
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _pip_install(reqs_file: Path, target: Path, flags: List[str], extra: List[str]) -> None:
|
|
153
|
+
cmd = (
|
|
154
|
+
[
|
|
155
|
+
sys.executable,
|
|
156
|
+
"-m",
|
|
157
|
+
"pip",
|
|
158
|
+
"install",
|
|
159
|
+
"--disable-pip-version-check",
|
|
160
|
+
"--no-compile",
|
|
161
|
+
"--quiet",
|
|
162
|
+
"--upgrade",
|
|
163
|
+
"-r",
|
|
164
|
+
str(reqs_file),
|
|
165
|
+
"--target",
|
|
166
|
+
str(target),
|
|
167
|
+
]
|
|
168
|
+
+ flags
|
|
169
|
+
+ extra
|
|
170
|
+
)
|
|
171
|
+
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
172
|
+
if proc.returncode != 0:
|
|
173
|
+
raise BuildError(
|
|
174
|
+
"pip install failed while vendoring dependencies:\n"
|
|
175
|
+
+ (proc.stderr or proc.stdout)[-4000:]
|
|
176
|
+
+ "\nHint: a dependency may have no Linux wheel for this runtime. "
|
|
177
|
+
"Pin a version that ships manylinux wheels, or set `package.pip_args`."
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _write_zip(build_dir: Path, zip_path: Path) -> int:
|
|
182
|
+
count = 0
|
|
183
|
+
tmp = zip_path.with_suffix(".zip.tmp")
|
|
184
|
+
with zipfile.ZipFile(tmp, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as zf:
|
|
185
|
+
for path in sorted(p for p in build_dir.rglob("*") if p.is_file()):
|
|
186
|
+
rel = path.relative_to(build_dir).as_posix()
|
|
187
|
+
if "/__pycache__/" in f"/{rel}" or rel.endswith(".pyc"):
|
|
188
|
+
continue
|
|
189
|
+
info = zipfile.ZipInfo(rel, date_time=_FIXED_DATE)
|
|
190
|
+
mode = 0o755 if os.access(path, os.X_OK) and path.suffix in ("", ".sh") else 0o644
|
|
191
|
+
info.external_attr = (0o100000 | mode) << 16
|
|
192
|
+
info.compress_type = zipfile.ZIP_DEFLATED
|
|
193
|
+
zf.writestr(info, path.read_bytes())
|
|
194
|
+
count += 1
|
|
195
|
+
os.replace(tmp, zip_path)
|
|
196
|
+
return count
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def digest(path: Path) -> tuple[str, str]:
|
|
200
|
+
h = hashlib.sha256()
|
|
201
|
+
with open(path, "rb") as fh:
|
|
202
|
+
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
203
|
+
h.update(chunk)
|
|
204
|
+
return h.hexdigest(), base64.b64encode(h.digest()).decode("ascii")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def build_artifact(project: Project, *, skip_deps: bool = False, log=print) -> Artifact: # noqa: ANN001
|
|
208
|
+
cfg = project.config
|
|
209
|
+
provider = project.provider
|
|
210
|
+
work = project.work_dir
|
|
211
|
+
build_dir = work / "build"
|
|
212
|
+
if build_dir.exists():
|
|
213
|
+
shutil.rmtree(build_dir)
|
|
214
|
+
build_dir.mkdir(parents=True)
|
|
215
|
+
|
|
216
|
+
source = (project.root / cfg.package.source).resolve()
|
|
217
|
+
files = collect_sources(source, cfg.package.include, cfg.package.exclude)
|
|
218
|
+
for f in files:
|
|
219
|
+
dest = build_dir / f.relative_to(source)
|
|
220
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
221
|
+
shutil.copy2(f, dest)
|
|
222
|
+
log(f" collected {len(files)} source file(s) from {source}")
|
|
223
|
+
|
|
224
|
+
_copy_runtime(build_dir)
|
|
225
|
+
(build_dir / "pskit_manifest.json").write_text(
|
|
226
|
+
json.dumps(project.bundle_manifest(), indent=2, sort_keys=True), "utf-8"
|
|
227
|
+
)
|
|
228
|
+
for rel, content in provider.shim_files(cfg).items():
|
|
229
|
+
dest = build_dir / rel
|
|
230
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
231
|
+
dest.write_text(content, "utf-8", newline="\n")
|
|
232
|
+
|
|
233
|
+
req_path = (project.root / cfg.package.requirements) if cfg.package.requirements else None
|
|
234
|
+
user_reqs = read_requirements(req_path)
|
|
235
|
+
vendor = provider.vendor_dependencies if cfg.package.vendor is None else cfg.package.vendor
|
|
236
|
+
final_reqs = provider.bundle_requirements(user_reqs)
|
|
237
|
+
if vendor:
|
|
238
|
+
if final_reqs and not skip_deps:
|
|
239
|
+
req_file = work / "requirements.vendor.txt"
|
|
240
|
+
req_file.write_text("\n".join(final_reqs) + "\n", "utf-8")
|
|
241
|
+
flags = provider.pip_platform(cfg.provider.runtime, cfg.provider.architecture)
|
|
242
|
+
log(f" vendoring {len(final_reqs)} requirement(s) for {cfg.provider.runtime}/{cfg.provider.architecture}")
|
|
243
|
+
_pip_install(req_file, build_dir, flags, cfg.package.pip_args)
|
|
244
|
+
else:
|
|
245
|
+
(build_dir / "requirements.txt").write_text("\n".join(final_reqs) + ("\n" if final_reqs else ""), "utf-8")
|
|
246
|
+
|
|
247
|
+
zip_path = work / "artifact.zip"
|
|
248
|
+
count = _write_zip(build_dir, zip_path)
|
|
249
|
+
sha_hex, sha_b64 = digest(zip_path)
|
|
250
|
+
size = zip_path.stat().st_size
|
|
251
|
+
log(f" artifact {zip_path.name}: {count} files, {size / 1024:.1f} KiB, sha256 {sha_hex[:12]}")
|
|
252
|
+
return Artifact(path=zip_path, sha256=sha_hex, sha256_b64=sha_b64, size=size, file_count=count, build_dir=build_dir)
|