py-serverless-kit 2.3.1__py3-none-any.whl → 2.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_serverless_kit/_version.py +1 -1
- py_serverless_kit/core/deployer.py +3 -2
- py_serverless_kit/core/function_split.py +19 -0
- py_serverless_kit/core/onprem_placement.py +15 -3
- py_serverless_kit/providers/onprem.py +54 -7
- py_serverless_kit/runtime/onprem_host.py +25 -0
- py_serverless_kit/runtime_js/package.json +1 -1
- py_serverless_kit/studio/studio.html +25 -25
- {py_serverless_kit-2.3.1.dist-info → py_serverless_kit-2.3.2.dist-info}/METADATA +1 -1
- {py_serverless_kit-2.3.1.dist-info → py_serverless_kit-2.3.2.dist-info}/RECORD +13 -13
- {py_serverless_kit-2.3.1.dist-info → py_serverless_kit-2.3.2.dist-info}/WHEEL +0 -0
- {py_serverless_kit-2.3.1.dist-info → py_serverless_kit-2.3.2.dist-info}/entry_points.txt +0 -0
- {py_serverless_kit-2.3.1.dist-info → py_serverless_kit-2.3.2.dist-info}/licenses/LICENSE +0 -0
py_serverless_kit/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "2.3.
|
|
1
|
+
__version__ = "2.3.2"
|
|
2
2
|
__author__ = "Asif Khan @ Kavion.ai"
|
|
@@ -297,7 +297,7 @@ class Deployer:
|
|
|
297
297
|
|
|
298
298
|
def _backends(self, data: Optional[Dict[str, Any]], groups: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
299
299
|
"""Where the load balancer sends this deployment's HTTP: a cluster's servers, or each host serving HTTP
|
|
300
|
-
(with placement,
|
|
300
|
+
(with placement, the default group's host: it passes a route on to the group that has it)."""
|
|
301
301
|
from urllib.parse import urlsplit
|
|
302
302
|
|
|
303
303
|
from .onprem_balancers import cluster_backends
|
|
@@ -308,7 +308,8 @@ class Deployer:
|
|
|
308
308
|
port = int(spec.get("port") or 8080)
|
|
309
309
|
if cluster:
|
|
310
310
|
return cluster_backends(get_cluster(self.project.root, str(cluster)), port)
|
|
311
|
-
|
|
311
|
+
# With placement only the default group passes routes on (placement.gateway): it is the entry point.
|
|
312
|
+
endpoints = [spec.get("endpoint")]
|
|
312
313
|
out = []
|
|
313
314
|
for endpoint in endpoints:
|
|
314
315
|
url = urlsplit(str(endpoint or ""))
|
|
@@ -327,6 +327,25 @@ def _matching_requirements(root: Path, modules: List[str]) -> Tuple[List[str], L
|
|
|
327
327
|
return found, unknown
|
|
328
328
|
|
|
329
329
|
|
|
330
|
+
def strip_functions(text: str, names: List[str]) -> Tuple[str, List[str]]:
|
|
331
|
+
"""``text`` (a Python module) without the functions ``names`` and the imports only they used: what a
|
|
332
|
+
placement group's copy of a shared code file keeps. A function that code left in the file still calls stays.
|
|
333
|
+
Returns the new text and the names removed."""
|
|
334
|
+
removed: List[str] = []
|
|
335
|
+
for name in names:
|
|
336
|
+
try:
|
|
337
|
+
tree = ast.parse(text)
|
|
338
|
+
except SyntaxError:
|
|
339
|
+
return text, removed
|
|
340
|
+
target = next((n for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
|
|
341
|
+
and n.name == name), None)
|
|
342
|
+
if target is None or _used_elsewhere(tree, target):
|
|
343
|
+
continue
|
|
344
|
+
text = _without(text.splitlines(), tree, target)
|
|
345
|
+
removed.append(name)
|
|
346
|
+
return text, removed
|
|
347
|
+
|
|
348
|
+
|
|
330
349
|
def _used_elsewhere(tree: ast.Module, target: ast.stmt) -> bool:
|
|
331
350
|
name = getattr(target, "name", "")
|
|
332
351
|
return any(name in _uses(node) for node in tree.body if node is not target)
|
|
@@ -63,9 +63,11 @@ def parse(raw: Any, functions: List[str], workflows: Optional[List[str]] = None)
|
|
|
63
63
|
return {}
|
|
64
64
|
if not isinstance(raw, dict):
|
|
65
65
|
raise PlacementError("options.placement must be a mapping with `groups` (and optionally `default`)")
|
|
66
|
-
unknown = sorted(set(raw) - {"default", "groups", "_groups"}) # _groups: resolved
|
|
66
|
+
unknown = sorted(set(raw) - {"default", "groups", "gateway", "_groups"}) # _groups: resolved (resolve())
|
|
67
67
|
if unknown:
|
|
68
|
-
raise PlacementError(f"unknown options.placement keys {unknown}; expected default, groups")
|
|
68
|
+
raise PlacementError(f"unknown options.placement keys {unknown}; expected default, groups, gateway")
|
|
69
|
+
if raw.get("gateway") not in (None, True, False):
|
|
70
|
+
raise PlacementError("options.placement.gateway is true or false")
|
|
69
71
|
groups_raw = raw.get("groups")
|
|
70
72
|
if not isinstance(groups_raw, dict) or not groups_raw:
|
|
71
73
|
raise PlacementError("options.placement.groups must name at least one group: {api: {server: app1}}")
|
|
@@ -130,7 +132,17 @@ def parse(raw: Any, functions: List[str], workflows: Optional[List[str]] = None)
|
|
|
130
132
|
function_group.setdefault(fn, default)
|
|
131
133
|
for name, group in groups.items():
|
|
132
134
|
group["functions"] = sorted(f for f, g in function_group.items() if g == name)
|
|
133
|
-
return {"default": default, "groups": groups, "function_group": function_group
|
|
135
|
+
return {"default": default, "groups": groups, "function_group": function_group,
|
|
136
|
+
**({"gateway": raw["gateway"]} if raw.get("gateway") is not None else {})}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def gateway(parsed: Dict[str, Any], options: Dict[str, Any]) -> bool:
|
|
140
|
+
"""Whether the default group answers every route, passing other groups' routes on (one URL for the project).
|
|
141
|
+
``placement.gateway``; unset: on with a load balancer (it needs one entry point), else off, so each server
|
|
142
|
+
answers only the routes of the functions it runs."""
|
|
143
|
+
if "gateway" in parsed:
|
|
144
|
+
return bool(parsed["gateway"])
|
|
145
|
+
return options.get("balancer") not in (None, "", False)
|
|
134
146
|
|
|
135
147
|
|
|
136
148
|
def group_stage(stage: str, group: str, default: str) -> str:
|
|
@@ -544,9 +544,10 @@ class OnPremProvider(Provider):
|
|
|
544
544
|
where = ", ".join(f"{n} ({groups.get(n)})" for n in names)
|
|
545
545
|
issues.append(Issue(
|
|
546
546
|
"warning",
|
|
547
|
-
f"{where} share {module.replace('.', '/')}.py
|
|
548
|
-
f"
|
|
549
|
-
f" {' '.join(names)} (studio: Functions -> Own
|
|
547
|
+
f"{where} share {module.replace('.', '/')}.py: each server gets a copy without the other groups'"
|
|
548
|
+
f" handlers, but with all of its helpers, and none has requirements of its own. Give each its own"
|
|
549
|
+
f" folder and requirements.txt: pskit function split {' '.join(names)} (studio: Functions -> Own"
|
|
550
|
+
f" folder)",
|
|
550
551
|
"provider.options.placement"))
|
|
551
552
|
return issues
|
|
552
553
|
|
|
@@ -819,9 +820,11 @@ class OnPremProvider(Provider):
|
|
|
819
820
|
}
|
|
820
821
|
|
|
821
822
|
def write_project(self, ctx: RenderContext, spec: Dict[str, Any], app_dir: str = APP_DIR,
|
|
822
|
-
exclude: Tuple[str, ...] = (), requirements: Tuple[str, ...] = ()
|
|
823
|
+
exclude: Tuple[str, ...] = (), requirements: Tuple[str, ...] = (),
|
|
824
|
+
strip: Optional[Dict[str, List[str]]] = None) -> str:
|
|
823
825
|
"""Write one host under ``<work_dir>/<app_dir>``. A placement group leaves out the sub-project folders of
|
|
824
|
-
functions placed elsewhere (``exclude``)
|
|
826
|
+
functions placed elsewhere (``exclude``), removes their handlers from the code files it shares with them
|
|
827
|
+
(``strip``: ``{module: [function names]}``) and installs its functions' ``requirements`` into its bundle."""
|
|
825
828
|
root = ctx.work_dir / app_dir
|
|
826
829
|
staging = root / "staging"
|
|
827
830
|
if staging.exists():
|
|
@@ -837,6 +840,9 @@ class OnPremProvider(Provider):
|
|
|
837
840
|
shutil.copytree(build, staging, ignore=ignore)
|
|
838
841
|
else: # pragma: no cover - the packager always sets build_dir
|
|
839
842
|
staging.mkdir(parents=True)
|
|
843
|
+
stripped = _strip_shared(staging, strip or {})
|
|
844
|
+
if stripped:
|
|
845
|
+
spec["stripped"] = stripped
|
|
840
846
|
if requirements and ctx.artifact.build_dir is not None and not ctx.artifact.skip_deps:
|
|
841
847
|
from ..packaging.builder import _pip_install
|
|
842
848
|
|
|
@@ -890,6 +896,8 @@ class OnPremProvider(Provider):
|
|
|
890
896
|
desc = f"pskit {spec['service']} ({spec['stage']})"
|
|
891
897
|
_write(root / "pskit.service", systemd_unit_text(desc, remote, user))
|
|
892
898
|
extra = "\n".join(("exclude",) + exclude + ("requirements",) + requirements) if (exclude or requirements) else ""
|
|
899
|
+
if strip:
|
|
900
|
+
extra += "\nstrip " + json.dumps(strip, sort_keys=True)
|
|
893
901
|
return hashlib.sha256((text + extra).encode("utf-8")).hexdigest()
|
|
894
902
|
|
|
895
903
|
def render_resources(self, doc: TerraformDoc, ctx: RenderContext) -> None:
|
|
@@ -1009,7 +1017,7 @@ class OnPremProvider(Provider):
|
|
|
1009
1017
|
|
|
1010
1018
|
def _render_groups(self, doc: TerraformDoc, ctx: RenderContext) -> None:
|
|
1011
1019
|
"""One host (Terraform ``terraform_data``) per placement group; core/onprem_placement.py."""
|
|
1012
|
-
from ..core.onprem_placement import app_dir, group_stage, owners, peer_token, resource_name
|
|
1020
|
+
from ..core.onprem_placement import app_dir, gateway, group_stage, owners, peer_token, resource_name
|
|
1013
1021
|
from ..packaging.subprojects import excluded_dirs, function_requirements
|
|
1014
1022
|
|
|
1015
1023
|
cfg = ctx.config
|
|
@@ -1047,16 +1055,18 @@ class OnPremProvider(Provider):
|
|
|
1047
1055
|
outputs: Dict[str, Any] = {}
|
|
1048
1056
|
fn_out: Dict[str, Any] = {}
|
|
1049
1057
|
for name, spec in specs.items():
|
|
1058
|
+
spec["gateway"] = name == default and gateway(parsed, cfg.provider.options)
|
|
1050
1059
|
spec.update({"group": name, "default_group": default, "groups": groups, "function_groups": function_groups,
|
|
1051
1060
|
"owners": found, "peer_token": token})
|
|
1052
1061
|
if name != default:
|
|
1053
1062
|
spec["deploy_stage"] = group_stage(cfg.provider.stage, name, default)
|
|
1054
1063
|
members = parsed["groups"][name]["functions"]
|
|
1055
1064
|
exclude = tuple(excluded_dirs(source, cfg.functions, members, cfg.language)) if many else ()
|
|
1065
|
+
strip = _shared_handlers(source, cfg, members) if many and cfg.language != "node" else {}
|
|
1056
1066
|
reqs = tuple(function_requirements(source, cfg.functions, members)) \
|
|
1057
1067
|
if many and cfg.language != "node" and self.vendor_dependencies else ()
|
|
1058
1068
|
folder = app_dir(name, default)
|
|
1059
|
-
digest = self.write_project(group_ctx[name], spec, folder, exclude, reqs)
|
|
1069
|
+
digest = self.write_project(group_ctx[name], spec, folder, exclude, reqs, strip)
|
|
1060
1070
|
self._host_resource(doc, ctx, resource_name(name, default), folder, digest, spec)
|
|
1061
1071
|
for f in members:
|
|
1062
1072
|
if f not in cfg.functions:
|
|
@@ -1111,6 +1121,43 @@ class OnPremProvider(Provider):
|
|
|
1111
1121
|
return doc
|
|
1112
1122
|
|
|
1113
1123
|
|
|
1124
|
+
def _shared_handlers(source: Path, cfg: ServiceConfig, members: List[str]) -> Dict[str, List[str]]:
|
|
1125
|
+
"""``{module: [handler functions]}`` of the functions placed in other groups whose handlers sit in a shared
|
|
1126
|
+
code file (not in their own sub-project folder): a group's copy of that file leaves them out."""
|
|
1127
|
+
from ..packaging.subprojects import function_dir
|
|
1128
|
+
|
|
1129
|
+
keep = set(members)
|
|
1130
|
+
out: Dict[str, List[str]] = {}
|
|
1131
|
+
for name, fn in sorted(cfg.functions.items()):
|
|
1132
|
+
if name in keep or not fn.enabled or is_generated_handler(fn.handler):
|
|
1133
|
+
continue
|
|
1134
|
+
if function_dir(source, fn.handler) is not None:
|
|
1135
|
+
continue # its own folder: left out whole (excluded_dirs)
|
|
1136
|
+
module, _, func = str(fn.handler).replace(":", ".").rpartition(".")
|
|
1137
|
+
if module and func:
|
|
1138
|
+
out.setdefault(module, []).append(func)
|
|
1139
|
+
return out
|
|
1140
|
+
|
|
1141
|
+
|
|
1142
|
+
def _strip_shared(staging: Path, strip: Dict[str, List[str]]) -> Dict[str, List[str]]:
|
|
1143
|
+
"""Remove other groups' handlers from this group's copies of shared code files. ``{file: [removed]}``."""
|
|
1144
|
+
from ..core.function_split import strip_functions
|
|
1145
|
+
|
|
1146
|
+
done: Dict[str, List[str]] = {}
|
|
1147
|
+
for module, names in sorted(strip.items()):
|
|
1148
|
+
parts = module.split(".")
|
|
1149
|
+
path = staging.joinpath(*parts, "__init__.py")
|
|
1150
|
+
if not path.is_file():
|
|
1151
|
+
path = staging.joinpath(*parts[:-1], parts[-1] + ".py")
|
|
1152
|
+
if not path.is_file():
|
|
1153
|
+
continue
|
|
1154
|
+
text, removed = strip_functions(path.read_text("utf-8"), sorted(names))
|
|
1155
|
+
if removed:
|
|
1156
|
+
path.write_bytes(text.encode("utf-8"))
|
|
1157
|
+
done[path.relative_to(staging).as_posix()] = removed
|
|
1158
|
+
return done
|
|
1159
|
+
|
|
1160
|
+
|
|
1114
1161
|
def _placement_of(options: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
1115
1162
|
raw = options.get("placement")
|
|
1116
1163
|
return raw if isinstance(raw, dict) and raw.get("groups") else None
|
|
@@ -1163,6 +1163,22 @@ class Host:
|
|
|
1163
1163
|
return self.peers.forward_http(group, event.http, self.peers.timeout(function))
|
|
1164
1164
|
return self.peers.invoke(function, event, group)
|
|
1165
1165
|
|
|
1166
|
+
@property
|
|
1167
|
+
def group(self) -> str:
|
|
1168
|
+
return str(self.spec.get("group") or "")
|
|
1169
|
+
|
|
1170
|
+
def served_elsewhere(self, function: str) -> Optional[Tuple[str, str]]:
|
|
1171
|
+
"""``(group, url)`` when an HTTP request for ``function`` reached a host that does not run it and does not
|
|
1172
|
+
pass routes on (only the default group with ``placement.gateway`` does)."""
|
|
1173
|
+
fns = self.spec.get("functions") or {}
|
|
1174
|
+
if self.peers is None or function in fns or self.spec.get("gateway"):
|
|
1175
|
+
return None
|
|
1176
|
+
group = self.peers.remote(function)
|
|
1177
|
+
if not group:
|
|
1178
|
+
return None
|
|
1179
|
+
url = str(((self.spec.get("groups") or {}).get(group) or {}).get("url") or "")
|
|
1180
|
+
return group, url
|
|
1181
|
+
|
|
1166
1182
|
def _route_remote(self, function: str, event: Event) -> Any:
|
|
1167
1183
|
"""``invoke()`` and workflow steps: functions of other groups go to their host."""
|
|
1168
1184
|
fns = self.spec.get("functions") or {}
|
|
@@ -2244,6 +2260,15 @@ class Host:
|
|
|
2244
2260
|
source_ip=self.client_address[0],
|
|
2245
2261
|
route=pattern,
|
|
2246
2262
|
)
|
|
2263
|
+
elsewhere = None if self._from_peer else host.served_elsewhere(function)
|
|
2264
|
+
if elsewhere is not None:
|
|
2265
|
+
# Its code is not on this server: say where it runs instead of passing it on.
|
|
2266
|
+
group, base = elsewhere
|
|
2267
|
+
return self._json(404, {
|
|
2268
|
+
"error": "Not Found",
|
|
2269
|
+
"message": f"{function} runs on group {group}, not on this server ({host.group})",
|
|
2270
|
+
"url": base.rstrip("/") + self.path,
|
|
2271
|
+
})
|
|
2247
2272
|
workflow = host.orchestrated(function)
|
|
2248
2273
|
if workflow is not None and self.command == "POST":
|
|
2249
2274
|
# Through the executor directly: tracked, async by default, no env lock held.
|