py-serverless-kit 2.3.1__py3-none-any.whl → 2.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2 +1,2 @@
1
- __version__ = "2.3.1"
1
+ __version__ = "2.3.2"
2
2
  __author__ = "Asif Khan @ Kavion.ai"
@@ -297,7 +297,7 @@ class Deployer:
297
297
 
298
298
  def _backends(self, data: Optional[Dict[str, Any]], groups: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
299
299
  """Where the load balancer sends this deployment's HTTP: a cluster's servers, or each host serving HTTP
300
- (with placement, every group's host: any of them passes a route on to the group that has it)."""
300
+ (with placement, the default group's host: it passes a route on to the group that has it)."""
301
301
  from urllib.parse import urlsplit
302
302
 
303
303
  from .onprem_balancers import cluster_backends
@@ -308,7 +308,8 @@ class Deployer:
308
308
  port = int(spec.get("port") or 8080)
309
309
  if cluster:
310
310
  return cluster_backends(get_cluster(self.project.root, str(cluster)), port)
311
- endpoints = [g.get("endpoint") for g in (groups or {}).values()] if groups else [spec.get("endpoint")]
311
+ # With placement only the default group passes routes on (placement.gateway): it is the entry point.
312
+ endpoints = [spec.get("endpoint")]
312
313
  out = []
313
314
  for endpoint in endpoints:
314
315
  url = urlsplit(str(endpoint or ""))
@@ -327,6 +327,25 @@ def _matching_requirements(root: Path, modules: List[str]) -> Tuple[List[str], L
327
327
  return found, unknown
328
328
 
329
329
 
330
+ def strip_functions(text: str, names: List[str]) -> Tuple[str, List[str]]:
331
+ """``text`` (a Python module) without the functions ``names`` and the imports only they used: what a
332
+ placement group's copy of a shared code file keeps. A function that code left in the file still calls stays.
333
+ Returns the new text and the names removed."""
334
+ removed: List[str] = []
335
+ for name in names:
336
+ try:
337
+ tree = ast.parse(text)
338
+ except SyntaxError:
339
+ return text, removed
340
+ target = next((n for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
341
+ and n.name == name), None)
342
+ if target is None or _used_elsewhere(tree, target):
343
+ continue
344
+ text = _without(text.splitlines(), tree, target)
345
+ removed.append(name)
346
+ return text, removed
347
+
348
+
330
349
  def _used_elsewhere(tree: ast.Module, target: ast.stmt) -> bool:
331
350
  name = getattr(target, "name", "")
332
351
  return any(name in _uses(node) for node in tree.body if node is not target)
@@ -63,9 +63,11 @@ def parse(raw: Any, functions: List[str], workflows: Optional[List[str]] = None)
63
63
  return {}
64
64
  if not isinstance(raw, dict):
65
65
  raise PlacementError("options.placement must be a mapping with `groups` (and optionally `default`)")
66
- unknown = sorted(set(raw) - {"default", "groups", "_groups"}) # _groups: resolved targets (resolve())
66
+ unknown = sorted(set(raw) - {"default", "groups", "gateway", "_groups"}) # _groups: resolved (resolve())
67
67
  if unknown:
68
- raise PlacementError(f"unknown options.placement keys {unknown}; expected default, groups")
68
+ raise PlacementError(f"unknown options.placement keys {unknown}; expected default, groups, gateway")
69
+ if raw.get("gateway") not in (None, True, False):
70
+ raise PlacementError("options.placement.gateway is true or false")
69
71
  groups_raw = raw.get("groups")
70
72
  if not isinstance(groups_raw, dict) or not groups_raw:
71
73
  raise PlacementError("options.placement.groups must name at least one group: {api: {server: app1}}")
@@ -130,7 +132,17 @@ def parse(raw: Any, functions: List[str], workflows: Optional[List[str]] = None)
130
132
  function_group.setdefault(fn, default)
131
133
  for name, group in groups.items():
132
134
  group["functions"] = sorted(f for f, g in function_group.items() if g == name)
133
- return {"default": default, "groups": groups, "function_group": function_group}
135
+ return {"default": default, "groups": groups, "function_group": function_group,
136
+ **({"gateway": raw["gateway"]} if raw.get("gateway") is not None else {})}
137
+
138
+
139
+ def gateway(parsed: Dict[str, Any], options: Dict[str, Any]) -> bool:
140
+ """Whether the default group answers every route, passing other groups' routes on (one URL for the project).
141
+ ``placement.gateway``; unset: on with a load balancer (it needs one entry point), else off, so each server
142
+ answers only the routes of the functions it runs."""
143
+ if "gateway" in parsed:
144
+ return bool(parsed["gateway"])
145
+ return options.get("balancer") not in (None, "", False)
134
146
 
135
147
 
136
148
  def group_stage(stage: str, group: str, default: str) -> str:
@@ -544,9 +544,10 @@ class OnPremProvider(Provider):
544
544
  where = ", ".join(f"{n} ({groups.get(n)})" for n in names)
545
545
  issues.append(Issue(
546
546
  "warning",
547
- f"{where} share {module.replace('.', '/')}.py, so every one of their servers gets all of it and the"
548
- f" project's requirements.txt. Give each its own folder and requirements: pskit function split"
549
- f" {' '.join(names)} (studio: Functions -> Own folder)",
547
+ f"{where} share {module.replace('.', '/')}.py: each server gets a copy without the other groups'"
548
+ f" handlers, but with all of its helpers, and none has requirements of its own. Give each its own"
549
+ f" folder and requirements.txt: pskit function split {' '.join(names)} (studio: Functions -> Own"
550
+ f" folder)",
550
551
  "provider.options.placement"))
551
552
  return issues
552
553
 
@@ -819,9 +820,11 @@ class OnPremProvider(Provider):
819
820
  }
820
821
 
821
822
  def write_project(self, ctx: RenderContext, spec: Dict[str, Any], app_dir: str = APP_DIR,
822
- exclude: Tuple[str, ...] = (), requirements: Tuple[str, ...] = ()) -> str:
823
+ exclude: Tuple[str, ...] = (), requirements: Tuple[str, ...] = (),
824
+ strip: Optional[Dict[str, List[str]]] = None) -> str:
823
825
  """Write one host under ``<work_dir>/<app_dir>``. A placement group leaves out the sub-project folders of
824
- functions placed elsewhere (``exclude``) and installs its functions' ``requirements`` into its bundle."""
826
+ functions placed elsewhere (``exclude``), removes their handlers from the code files it shares with them
827
+ (``strip``: ``{module: [function names]}``) and installs its functions' ``requirements`` into its bundle."""
825
828
  root = ctx.work_dir / app_dir
826
829
  staging = root / "staging"
827
830
  if staging.exists():
@@ -837,6 +840,9 @@ class OnPremProvider(Provider):
837
840
  shutil.copytree(build, staging, ignore=ignore)
838
841
  else: # pragma: no cover - the packager always sets build_dir
839
842
  staging.mkdir(parents=True)
843
+ stripped = _strip_shared(staging, strip or {})
844
+ if stripped:
845
+ spec["stripped"] = stripped
840
846
  if requirements and ctx.artifact.build_dir is not None and not ctx.artifact.skip_deps:
841
847
  from ..packaging.builder import _pip_install
842
848
 
@@ -890,6 +896,8 @@ class OnPremProvider(Provider):
890
896
  desc = f"pskit {spec['service']} ({spec['stage']})"
891
897
  _write(root / "pskit.service", systemd_unit_text(desc, remote, user))
892
898
  extra = "\n".join(("exclude",) + exclude + ("requirements",) + requirements) if (exclude or requirements) else ""
899
+ if strip:
900
+ extra += "\nstrip " + json.dumps(strip, sort_keys=True)
893
901
  return hashlib.sha256((text + extra).encode("utf-8")).hexdigest()
894
902
 
895
903
  def render_resources(self, doc: TerraformDoc, ctx: RenderContext) -> None:
@@ -1009,7 +1017,7 @@ class OnPremProvider(Provider):
1009
1017
 
1010
1018
  def _render_groups(self, doc: TerraformDoc, ctx: RenderContext) -> None:
1011
1019
  """One host (Terraform ``terraform_data``) per placement group; core/onprem_placement.py."""
1012
- from ..core.onprem_placement import app_dir, group_stage, owners, peer_token, resource_name
1020
+ from ..core.onprem_placement import app_dir, gateway, group_stage, owners, peer_token, resource_name
1013
1021
  from ..packaging.subprojects import excluded_dirs, function_requirements
1014
1022
 
1015
1023
  cfg = ctx.config
@@ -1047,16 +1055,18 @@ class OnPremProvider(Provider):
1047
1055
  outputs: Dict[str, Any] = {}
1048
1056
  fn_out: Dict[str, Any] = {}
1049
1057
  for name, spec in specs.items():
1058
+ spec["gateway"] = name == default and gateway(parsed, cfg.provider.options)
1050
1059
  spec.update({"group": name, "default_group": default, "groups": groups, "function_groups": function_groups,
1051
1060
  "owners": found, "peer_token": token})
1052
1061
  if name != default:
1053
1062
  spec["deploy_stage"] = group_stage(cfg.provider.stage, name, default)
1054
1063
  members = parsed["groups"][name]["functions"]
1055
1064
  exclude = tuple(excluded_dirs(source, cfg.functions, members, cfg.language)) if many else ()
1065
+ strip = _shared_handlers(source, cfg, members) if many and cfg.language != "node" else {}
1056
1066
  reqs = tuple(function_requirements(source, cfg.functions, members)) \
1057
1067
  if many and cfg.language != "node" and self.vendor_dependencies else ()
1058
1068
  folder = app_dir(name, default)
1059
- digest = self.write_project(group_ctx[name], spec, folder, exclude, reqs)
1069
+ digest = self.write_project(group_ctx[name], spec, folder, exclude, reqs, strip)
1060
1070
  self._host_resource(doc, ctx, resource_name(name, default), folder, digest, spec)
1061
1071
  for f in members:
1062
1072
  if f not in cfg.functions:
@@ -1111,6 +1121,43 @@ class OnPremProvider(Provider):
1111
1121
  return doc
1112
1122
 
1113
1123
 
1124
+ def _shared_handlers(source: Path, cfg: ServiceConfig, members: List[str]) -> Dict[str, List[str]]:
1125
+ """``{module: [handler functions]}`` of the functions placed in other groups whose handlers sit in a shared
1126
+ code file (not in their own sub-project folder): a group's copy of that file leaves them out."""
1127
+ from ..packaging.subprojects import function_dir
1128
+
1129
+ keep = set(members)
1130
+ out: Dict[str, List[str]] = {}
1131
+ for name, fn in sorted(cfg.functions.items()):
1132
+ if name in keep or not fn.enabled or is_generated_handler(fn.handler):
1133
+ continue
1134
+ if function_dir(source, fn.handler) is not None:
1135
+ continue # its own folder: left out whole (excluded_dirs)
1136
+ module, _, func = str(fn.handler).replace(":", ".").rpartition(".")
1137
+ if module and func:
1138
+ out.setdefault(module, []).append(func)
1139
+ return out
1140
+
1141
+
1142
+ def _strip_shared(staging: Path, strip: Dict[str, List[str]]) -> Dict[str, List[str]]:
1143
+ """Remove other groups' handlers from this group's copies of shared code files. ``{file: [removed]}``."""
1144
+ from ..core.function_split import strip_functions
1145
+
1146
+ done: Dict[str, List[str]] = {}
1147
+ for module, names in sorted(strip.items()):
1148
+ parts = module.split(".")
1149
+ path = staging.joinpath(*parts, "__init__.py")
1150
+ if not path.is_file():
1151
+ path = staging.joinpath(*parts[:-1], parts[-1] + ".py")
1152
+ if not path.is_file():
1153
+ continue
1154
+ text, removed = strip_functions(path.read_text("utf-8"), sorted(names))
1155
+ if removed:
1156
+ path.write_bytes(text.encode("utf-8"))
1157
+ done[path.relative_to(staging).as_posix()] = removed
1158
+ return done
1159
+
1160
+
1114
1161
  def _placement_of(options: Dict[str, Any]) -> Optional[Dict[str, Any]]:
1115
1162
  raw = options.get("placement")
1116
1163
  return raw if isinstance(raw, dict) and raw.get("groups") else None
@@ -1163,6 +1163,22 @@ class Host:
1163
1163
  return self.peers.forward_http(group, event.http, self.peers.timeout(function))
1164
1164
  return self.peers.invoke(function, event, group)
1165
1165
 
1166
+ @property
1167
+ def group(self) -> str:
1168
+ return str(self.spec.get("group") or "")
1169
+
1170
+ def served_elsewhere(self, function: str) -> Optional[Tuple[str, str]]:
1171
+ """``(group, url)`` when an HTTP request for ``function`` reached a host that does not run it and does not
1172
+ pass routes on (only the default group with ``placement.gateway`` does)."""
1173
+ fns = self.spec.get("functions") or {}
1174
+ if self.peers is None or function in fns or self.spec.get("gateway"):
1175
+ return None
1176
+ group = self.peers.remote(function)
1177
+ if not group:
1178
+ return None
1179
+ url = str(((self.spec.get("groups") or {}).get(group) or {}).get("url") or "")
1180
+ return group, url
1181
+
1166
1182
  def _route_remote(self, function: str, event: Event) -> Any:
1167
1183
  """``invoke()`` and workflow steps: functions of other groups go to their host."""
1168
1184
  fns = self.spec.get("functions") or {}
@@ -2244,6 +2260,15 @@ class Host:
2244
2260
  source_ip=self.client_address[0],
2245
2261
  route=pattern,
2246
2262
  )
2263
+ elsewhere = None if self._from_peer else host.served_elsewhere(function)
2264
+ if elsewhere is not None:
2265
+ # Its code is not on this server: say where it runs instead of passing it on.
2266
+ group, base = elsewhere
2267
+ return self._json(404, {
2268
+ "error": "Not Found",
2269
+ "message": f"{function} runs on group {group}, not on this server ({host.group})",
2270
+ "url": base.rstrip("/") + self.path,
2271
+ })
2247
2272
  workflow = host.orchestrated(function)
2248
2273
  if workflow is not None and self.command == "POST":
2249
2274
  # Through the executor directly: tracked, async by default, no env lock held.
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pskit",
3
- "version": "2.3.1",
3
+ "version": "2.3.2",
4
4
  "description": "py-serverless-kit runtime for Node.js and TypeScript handlers (vendored into every bundle).",
5
5
  "main": "index.js",
6
6
  "types": "index.d.ts",