py-serverless-kit 2.3.0__py3-none-any.whl → 2.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_serverless_kit/_version.py +1 -1
- py_serverless_kit/cli/main.py +50 -0
- py_serverless_kit/core/deployer.py +3 -2
- py_serverless_kit/core/function_split.py +378 -0
- py_serverless_kit/core/onprem_placement.py +25 -5
- py_serverless_kit/providers/onprem.py +80 -7
- py_serverless_kit/runtime/onprem_host.py +44 -3
- py_serverless_kit/runtime_js/package.json +1 -1
- py_serverless_kit/studio/server.py +17 -0
- py_serverless_kit/studio/studio.html +51 -45
- {py_serverless_kit-2.3.0.dist-info → py_serverless_kit-2.3.2.dist-info}/METADATA +1 -1
- {py_serverless_kit-2.3.0.dist-info → py_serverless_kit-2.3.2.dist-info}/RECORD +15 -14
- {py_serverless_kit-2.3.0.dist-info → py_serverless_kit-2.3.2.dist-info}/WHEEL +0 -0
- {py_serverless_kit-2.3.0.dist-info → py_serverless_kit-2.3.2.dist-info}/entry_points.txt +0 -0
- {py_serverless_kit-2.3.0.dist-info → py_serverless_kit-2.3.2.dist-info}/licenses/LICENSE +0 -0
py_serverless_kit/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "2.3.
|
|
1
|
+
__version__ = "2.3.2"
|
|
2
2
|
__author__ = "Asif Khan @ Kavion.ai"
|
py_serverless_kit/cli/main.py
CHANGED
|
@@ -150,6 +150,7 @@ Operate
|
|
|
150
150
|
pskit cluster list
|
|
151
151
|
pskit cluster remove prod --yes
|
|
152
152
|
firewall Ports each on-prem server must open pskit firewall --server edge2
|
|
153
|
+
function split: a function into its own folder pskit function split get-item
|
|
153
154
|
lb algorithms | create | update | list | status | attach | detach | routes | sync | logs | remove | forget
|
|
154
155
|
pskit lb algorithms
|
|
155
156
|
pskit lb create edge --server lb1 --algorithm least_time
|
|
@@ -1818,6 +1819,55 @@ def cluster_balancer_logs(name: str, lines: int, config_path: Optional[str]) ->
|
|
|
1818
1819
|
click.echo(_cluster_call(balancer_logs, _cluster_root(config_path), name, lines).rstrip("\n"))
|
|
1819
1820
|
|
|
1820
1821
|
|
|
1822
|
+
# ----------------------------------------------------------------------------- functions
|
|
1823
|
+
@cli.group("function", short_help="Give a function its own folder, code and requirements.txt.")
|
|
1824
|
+
def function_group() -> None:
|
|
1825
|
+
"""Functions as sub-projects: functions/<name>/ with their own code and requirements.txt, so a server of
|
|
1826
|
+
provider.options.placement gets only the code and dependencies of the functions it runs.
|
|
1827
|
+
|
|
1828
|
+
\b
|
|
1829
|
+
Examples:
|
|
1830
|
+
pskit function split get-item
|
|
1831
|
+
pskit function split --all --dry-run
|
|
1832
|
+
"""
|
|
1833
|
+
|
|
1834
|
+
|
|
1835
|
+
@function_group.command("split")
|
|
1836
|
+
@click.argument("names", nargs=-1)
|
|
1837
|
+
@click.option("--all", "split_all", is_flag=True, help="Every function that shares a code file with another.")
|
|
1838
|
+
@click.option("--dry-run", is_flag=True, help="Only show what would move.")
|
|
1839
|
+
@_CONFIG_OPTION
|
|
1840
|
+
def function_split(names: Tuple[str, ...], split_all: bool, dry_run: bool, config_path: Optional[str]) -> None:
|
|
1841
|
+
"""Move functions out of a shared code file into functions/<name>/handler.py with their own requirements.txt.
|
|
1842
|
+
|
|
1843
|
+
\b
|
|
1844
|
+
Examples:
|
|
1845
|
+
pskit function split get-item
|
|
1846
|
+
pskit function split --all --dry-run
|
|
1847
|
+
|
|
1848
|
+
The helpers and imports the function uses are copied; the function is removed from the shared file unless code
|
|
1849
|
+
there still calls it; its handler in pskit.yml is updated; requirements.txt gets the project requirements its
|
|
1850
|
+
imports name.
|
|
1851
|
+
"""
|
|
1852
|
+
from ..core import function_split as fsplit
|
|
1853
|
+
|
|
1854
|
+
root = _cluster_root(config_path)
|
|
1855
|
+
if split_all:
|
|
1856
|
+
names = tuple(n for group in fsplit.shared_in_project(root).values() for n in group)
|
|
1857
|
+
if not names:
|
|
1858
|
+
echo("Every function already has its own folder (or its own code file).")
|
|
1859
|
+
return
|
|
1860
|
+
if not names:
|
|
1861
|
+
raise click.UsageError("name a function, or use --all")
|
|
1862
|
+
for name in names:
|
|
1863
|
+
try:
|
|
1864
|
+
result = (fsplit.plan if dry_run else fsplit.split)(root, name)
|
|
1865
|
+
except fsplit.SplitError as exc:
|
|
1866
|
+
raise click.ClickException(str(exc)) from None
|
|
1867
|
+
for line in fsplit.describe(result):
|
|
1868
|
+
echo(("would move " if dry_run and line == fsplit.describe(result)[0] else "") + line)
|
|
1869
|
+
|
|
1870
|
+
|
|
1821
1871
|
# ----------------------------------------------------------------------------- firewall
|
|
1822
1872
|
@cli.command(short_help="Ports each on-prem server must open (inbound / outbound), with server names.")
|
|
1823
1873
|
@click.option("--server", "only", default="", help="Only this server (name or address).")
|
|
@@ -297,7 +297,7 @@ class Deployer:
|
|
|
297
297
|
|
|
298
298
|
def _backends(self, data: Optional[Dict[str, Any]], groups: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
299
299
|
"""Where the load balancer sends this deployment's HTTP: a cluster's servers, or each host serving HTTP
|
|
300
|
-
(with placement,
|
|
300
|
+
(with placement, the default group's host: it passes a route on to the group that has it)."""
|
|
301
301
|
from urllib.parse import urlsplit
|
|
302
302
|
|
|
303
303
|
from .onprem_balancers import cluster_backends
|
|
@@ -308,7 +308,8 @@ class Deployer:
|
|
|
308
308
|
port = int(spec.get("port") or 8080)
|
|
309
309
|
if cluster:
|
|
310
310
|
return cluster_backends(get_cluster(self.project.root, str(cluster)), port)
|
|
311
|
-
|
|
311
|
+
# With placement only the default group passes routes on (placement.gateway): it is the entry point.
|
|
312
|
+
endpoints = [spec.get("endpoint")]
|
|
312
313
|
out = []
|
|
313
314
|
for endpoint in endpoints:
|
|
314
315
|
url = urlsplit(str(endpoint or ""))
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
"""Move a function that shares a code file with others into its own sub-project folder.
|
|
2
|
+
|
|
3
|
+
``pskit function split get-item`` (and the studio's "Own folder" button) turns::
|
|
4
|
+
|
|
5
|
+
app/handlers.py # hello, get_item, create_item side by side
|
|
6
|
+
requirements.txt
|
|
7
|
+
|
|
8
|
+
into::
|
|
9
|
+
|
|
10
|
+
app/handlers.py # hello, create_item
|
|
11
|
+
functions/get_item/__init__.py
|
|
12
|
+
functions/get_item/handler.py # get_item, the helpers and imports it uses
|
|
13
|
+
functions/get_item/requirements.txt # the project requirements its imports name
|
|
14
|
+
|
|
15
|
+
and points ``functions.get-item.handler`` at ``functions.get_item.handler.get_item``. With
|
|
16
|
+
``provider.options.placement`` the server that runs get-item then gets only that folder and its requirements
|
|
17
|
+
(``packaging/subprojects.py``); other servers do not get its code.
|
|
18
|
+
|
|
19
|
+
Helpers used by the function are copied (other functions may still use them); the function itself is removed from
|
|
20
|
+
the shared file unless something left there still calls it. Python only: a Node.js handler is moved by hand.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import ast
|
|
26
|
+
import re
|
|
27
|
+
import sys
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any, Dict, List, Set, Tuple
|
|
30
|
+
|
|
31
|
+
import yaml
|
|
32
|
+
|
|
33
|
+
from ..packaging.subprojects import PYTHON_MARKER, function_dir
|
|
34
|
+
|
|
35
|
+
_HANDLER_RE = re.compile(r"^([A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*)*)[.:]([A-Za-z_]\w*)$")
|
|
36
|
+
# Import name -> distribution name, for the common ones whose names differ.
|
|
37
|
+
_DISTRIBUTIONS = {"pil": "pillow", "yaml": "pyyaml", "cv2": "opencv-python", "sklearn": "scikit-learn",
|
|
38
|
+
"bs4": "beautifulsoup4", "dateutil": "python-dateutil", "jwt": "pyjwt", "dotenv": "python-dotenv",
|
|
39
|
+
"google": "google-cloud", "magic": "python-magic", "docx": "python-docx"}
|
|
40
|
+
_REQ_NAME = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class SplitError(ValueError):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _config_path(root: Path) -> Path:
|
|
48
|
+
for name in ("pskit.yml", "pskit.yaml"):
|
|
49
|
+
if (root / name).is_file():
|
|
50
|
+
return root / name
|
|
51
|
+
raise SplitError("no pskit.yml in this folder")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _norm(name: str) -> str:
|
|
55
|
+
return re.sub(r"[-_.]+", "-", name).lower()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def folder_name(function: str) -> str:
|
|
59
|
+
return re.sub(r"\W", "_", function.replace("-", "_")).lower()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def plan(root: Path, function: str) -> Dict[str, Any]:
|
|
63
|
+
"""What a split would do, without writing: ``{function, from, to, handler, requirements, keeps, shared}``."""
|
|
64
|
+
return _split(Path(root), function, write=False)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def split(root: Path, function: str) -> Dict[str, Any]:
|
|
68
|
+
return _split(Path(root), function, write=True)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def shared_functions(root: Path, functions: Dict[str, Any], source: Path) -> Dict[str, List[str]]:
|
|
72
|
+
"""``{code file: [functions]}`` for files that hold the handlers of more than one function and are not a
|
|
73
|
+
sub-project (so every server of a placement gets all of them)."""
|
|
74
|
+
out: Dict[str, List[str]] = {}
|
|
75
|
+
for name, fn in functions.items():
|
|
76
|
+
if not getattr(fn, "enabled", True):
|
|
77
|
+
continue
|
|
78
|
+
handler = str(getattr(fn, "handler", "") or "")
|
|
79
|
+
if function_dir(source, handler) is not None:
|
|
80
|
+
continue
|
|
81
|
+
match = _HANDLER_RE.match(handler)
|
|
82
|
+
if not match:
|
|
83
|
+
continue
|
|
84
|
+
out.setdefault(match.group(1), []).append(name)
|
|
85
|
+
return {module: sorted(names) for module, names in out.items() if len(names) > 1}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def shared_in_project(root: Path) -> Dict[str, List[str]]:
|
|
89
|
+
"""``shared_functions`` of the project in ``root`` (pskit.yml as written)."""
|
|
90
|
+
raw = yaml.safe_load(_config_path(Path(root)).read_text("utf-8")) or {}
|
|
91
|
+
package = raw.get("package") or {}
|
|
92
|
+
source = (Path(root) / str(package.get("source") or ".")).resolve() if isinstance(package, dict) else Path(root)
|
|
93
|
+
functions = {k: _Fn(v) for k, v in (raw.get("functions") or {}).items()}
|
|
94
|
+
return shared_functions(Path(root), functions, source)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class _Fn:
|
|
98
|
+
def __init__(self, body: Any) -> None:
|
|
99
|
+
body = body if isinstance(body, dict) else {}
|
|
100
|
+
self.handler = str(body.get("handler") or "")
|
|
101
|
+
self.enabled = body.get("enabled", True) is not False
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _split(root: Path, function: str, write: bool) -> Dict[str, Any]:
|
|
105
|
+
path = _config_path(root)
|
|
106
|
+
raw = yaml.safe_load(path.read_text("utf-8")) or {}
|
|
107
|
+
functions = raw.get("functions") or {}
|
|
108
|
+
if function not in functions:
|
|
109
|
+
raise SplitError(f"no function {function!r} in {path.name} (functions: {', '.join(functions) or 'none'})")
|
|
110
|
+
runtime = str((raw.get("provider") or {}).get("runtime") or "")
|
|
111
|
+
if runtime.startswith("nodejs"):
|
|
112
|
+
raise SplitError("moving a Node.js handler is not automated yet: move it to functions/<name>/index.js with a"
|
|
113
|
+
" package.json and update its handler in pskit.yml")
|
|
114
|
+
handler = str((functions[function] or {}).get("handler") or "")
|
|
115
|
+
match = _HANDLER_RE.match(handler)
|
|
116
|
+
if not match:
|
|
117
|
+
raise SplitError(f"function {function} has no handler like package.module.function ({handler!r})")
|
|
118
|
+
module, func = match.group(1), match.group(2)
|
|
119
|
+
package = raw.get("package") or {}
|
|
120
|
+
source = (root / str(package.get("source") or ".")).resolve() if isinstance(package, dict) else root.resolve()
|
|
121
|
+
if function_dir(source, handler) is not None:
|
|
122
|
+
raise SplitError(f"{function} already has its own folder ({function_dir(source, handler)})")
|
|
123
|
+
parts = module.split(".")
|
|
124
|
+
file = source.joinpath(*parts, "__init__.py")
|
|
125
|
+
if not file.is_file():
|
|
126
|
+
file = source.joinpath(*parts[:-1], parts[-1] + ".py")
|
|
127
|
+
if not file.is_file():
|
|
128
|
+
raise SplitError(f"cannot find the code of {handler} ({file.relative_to(root).as_posix()})")
|
|
129
|
+
text = file.read_text("utf-8")
|
|
130
|
+
try:
|
|
131
|
+
tree = ast.parse(text)
|
|
132
|
+
except SyntaxError as exc:
|
|
133
|
+
raise SplitError(f"{file.name} has a syntax error at line {exc.lineno}; fix it first") from None
|
|
134
|
+
target = next((n for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == func),
|
|
135
|
+
None)
|
|
136
|
+
if target is None:
|
|
137
|
+
raise SplitError(f"{file.relative_to(root).as_posix()} has no function {func}")
|
|
138
|
+
|
|
139
|
+
folder = folder_name(function)
|
|
140
|
+
dest = source / "functions" / folder
|
|
141
|
+
if (dest / "handler.py").exists():
|
|
142
|
+
raise SplitError(f"{dest.relative_to(root).as_posix()}/handler.py already exists")
|
|
143
|
+
lines = text.splitlines()
|
|
144
|
+
package_parts = parts if file.name == "__init__.py" else parts[:-1]
|
|
145
|
+
imports, helpers = _needed(tree, target)
|
|
146
|
+
used = set().union(*(_uses(n) for n in helpers + [target]))
|
|
147
|
+
body: List[str] = ['"""' + f"Function {function} (moved from {file.relative_to(root).as_posix()})." + '"""', ""]
|
|
148
|
+
body += [_absolute(_import_code(node, used), node, package_parts) for node in imports]
|
|
149
|
+
if imports:
|
|
150
|
+
body.append("")
|
|
151
|
+
for node in helpers + [target]:
|
|
152
|
+
body += ["", _segment(lines, node), ""]
|
|
153
|
+
new_code = "\n".join(body).rstrip() + "\n"
|
|
154
|
+
new_code = re.sub(r"\n{4,}", "\n\n\n", new_code)
|
|
155
|
+
|
|
156
|
+
third_party = _third_party(imports, source)
|
|
157
|
+
reqs, unknown = _matching_requirements(root, third_party)
|
|
158
|
+
req_text = ("# Dependencies of this function only. pskit installs them with the function: on premises with\n"
|
|
159
|
+
"# provider.options.placement only on the server this function runs on.\n")
|
|
160
|
+
req_text += "".join(f"{line}\n" for line in reqs)
|
|
161
|
+
if unknown:
|
|
162
|
+
req_text += f"# Also imported, add its package if it is not in the standard library: {', '.join(unknown)}\n"
|
|
163
|
+
|
|
164
|
+
still_used = _used_elsewhere(tree, target)
|
|
165
|
+
new_handler = f"functions.{folder}.handler.{func}"
|
|
166
|
+
result = {"function": function, "from": file.relative_to(root).as_posix(),
|
|
167
|
+
"to": (dest / "handler.py").relative_to(root).as_posix(), "handler": new_handler,
|
|
168
|
+
"requirements": reqs, "unknown_imports": unknown, "helpers": [_name(n) for n in helpers],
|
|
169
|
+
"kept_in_source": still_used}
|
|
170
|
+
if not write:
|
|
171
|
+
return result
|
|
172
|
+
for pkg in (source / "functions", dest):
|
|
173
|
+
pkg.mkdir(parents=True, exist_ok=True)
|
|
174
|
+
init = pkg / "__init__.py"
|
|
175
|
+
if not init.exists():
|
|
176
|
+
init.write_bytes(b"")
|
|
177
|
+
(dest / "handler.py").write_bytes(new_code.encode("utf-8"))
|
|
178
|
+
if not (dest / PYTHON_MARKER).exists():
|
|
179
|
+
(dest / PYTHON_MARKER).write_bytes(req_text.encode("utf-8"))
|
|
180
|
+
if not still_used:
|
|
181
|
+
file.write_bytes(_without(lines, tree, target).encode("utf-8"))
|
|
182
|
+
_set_handler(path, function, handler, new_handler)
|
|
183
|
+
return result
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _import_code(node: ast.stmt, used: Set[str]) -> str:
|
|
187
|
+
"""``node`` with only the names in ``used`` ("" when none is)."""
|
|
188
|
+
assert isinstance(node, (ast.Import, ast.ImportFrom))
|
|
189
|
+
if isinstance(node, ast.ImportFrom) and node.module == "__future__":
|
|
190
|
+
return f"from __future__ import {', '.join(a.name for a in node.names)}"
|
|
191
|
+
names = [a for a in node.names if (a.asname or a.name).split(".")[0] in used]
|
|
192
|
+
if not names:
|
|
193
|
+
return ""
|
|
194
|
+
listed = ", ".join(a.name + (f" as {a.asname}" if a.asname else "") for a in names)
|
|
195
|
+
if isinstance(node, ast.ImportFrom):
|
|
196
|
+
return f"from {'.' * node.level}{node.module or ''} import {listed}"
|
|
197
|
+
return f"import {listed}"
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _without(lines: List[str], tree: ast.Module, target: ast.stmt) -> str:
|
|
201
|
+
"""The shared file without ``target``, and without the imported names only it used."""
|
|
202
|
+
rest = [n for n in tree.body if n is not target and not isinstance(n, (ast.Import, ast.ImportFrom))]
|
|
203
|
+
still: Set[str] = set().union(*(_uses(n) for n in rest)) if rest else set()
|
|
204
|
+
only_target = _uses(target) - still
|
|
205
|
+
edits: List[Tuple[int, int, str]] = [(*_span(target), "")]
|
|
206
|
+
for node in tree.body:
|
|
207
|
+
if not isinstance(node, (ast.Import, ast.ImportFrom)) or not (_bound(node) & only_target):
|
|
208
|
+
continue
|
|
209
|
+
keep = {(a.asname or a.name).split(".")[0] for a in node.names} - only_target
|
|
210
|
+
edits.append((*_span(node), _import_code(node, keep)))
|
|
211
|
+
out = list(lines)
|
|
212
|
+
for start, end, code in sorted(edits, reverse=True):
|
|
213
|
+
out[start:end] = [code] if code else []
|
|
214
|
+
return re.sub(r"\n{4,}", "\n\n\n", "\n".join(out)).rstrip() + "\n"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _name(node: ast.stmt) -> str:
|
|
218
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
219
|
+
return node.name
|
|
220
|
+
return ", ".join(sorted(_bound(node)))
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _bound(node: ast.stmt) -> Set[str]:
|
|
224
|
+
"""Names a module-level statement defines."""
|
|
225
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
226
|
+
return {node.name}
|
|
227
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
228
|
+
return {(a.asname or a.name).split(".")[0] for a in node.names}
|
|
229
|
+
targets: List[ast.AST] = []
|
|
230
|
+
if isinstance(node, ast.Assign):
|
|
231
|
+
targets = list(node.targets)
|
|
232
|
+
elif isinstance(node, (ast.AnnAssign, ast.AugAssign)):
|
|
233
|
+
targets = [node.target]
|
|
234
|
+
return {n.id for t in targets for n in ast.walk(t) if isinstance(n, ast.Name)}
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _uses(node: ast.AST) -> Set[str]:
|
|
238
|
+
return {n.id for n in ast.walk(node) if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load)}
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _needed(tree: ast.Module, target: ast.stmt) -> Tuple[List[ast.stmt], List[ast.stmt]]:
|
|
242
|
+
"""The imports and module-level helpers ``target`` uses, directly or through those helpers (in file order)."""
|
|
243
|
+
defs: Dict[str, ast.stmt] = {}
|
|
244
|
+
for node in tree.body:
|
|
245
|
+
if node is target or isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
246
|
+
continue
|
|
247
|
+
for name in _bound(node):
|
|
248
|
+
defs.setdefault(name, node)
|
|
249
|
+
wanted: List[ast.stmt] = []
|
|
250
|
+
todo = [target]
|
|
251
|
+
seen_names: Set[str] = set()
|
|
252
|
+
while todo:
|
|
253
|
+
node = todo.pop()
|
|
254
|
+
for name in sorted(_uses(node) - seen_names):
|
|
255
|
+
seen_names.add(name)
|
|
256
|
+
helper = defs.get(name)
|
|
257
|
+
if helper is not None and helper not in wanted:
|
|
258
|
+
wanted.append(helper)
|
|
259
|
+
todo.append(helper)
|
|
260
|
+
helpers = [n for n in tree.body if n in wanted]
|
|
261
|
+
imports: List[ast.stmt] = [n for n in tree.body if isinstance(n, (ast.Import, ast.ImportFrom)) and (
|
|
262
|
+
_bound(n) & seen_names or (isinstance(n, ast.ImportFrom) and n.module == "__future__"))]
|
|
263
|
+
return imports, helpers
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _span(node: ast.stmt) -> Tuple[int, int]:
|
|
267
|
+
"""0-based [start, end) line span of a statement with its decorators."""
|
|
268
|
+
first = min([node.lineno] + [d.lineno for d in getattr(node, "decorator_list", [])])
|
|
269
|
+
return first - 1, int(getattr(node, "end_lineno", node.lineno))
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _segment(lines: List[str], node: ast.stmt) -> str:
|
|
273
|
+
start, end = _span(node)
|
|
274
|
+
return "\n".join(lines[start:end])
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _absolute(code: str, node: ast.stmt, package: List[str]) -> str:
|
|
278
|
+
"""``from .util import x`` in app/handlers.py -> ``from app.util import x`` (the new file lives elsewhere)."""
|
|
279
|
+
if not isinstance(node, ast.ImportFrom) or not node.level:
|
|
280
|
+
return code
|
|
281
|
+
base = package[: len(package) - (node.level - 1)] if node.level > 1 else list(package)
|
|
282
|
+
module = ".".join(base + ([node.module] if node.module else []))
|
|
283
|
+
names = ", ".join(a.name + (f" as {a.asname}" if a.asname else "") for a in node.names)
|
|
284
|
+
return f"from {module} import {names}"
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _third_party(imports: List[ast.stmt], source: Path) -> List[str]:
|
|
288
|
+
stdlib = set(getattr(sys, "stdlib_module_names", ()))
|
|
289
|
+
out: List[str] = []
|
|
290
|
+
for node in imports:
|
|
291
|
+
if isinstance(node, ast.ImportFrom):
|
|
292
|
+
if node.level or not node.module:
|
|
293
|
+
continue
|
|
294
|
+
tops = [node.module.split(".")[0]]
|
|
295
|
+
else:
|
|
296
|
+
tops = [a.name.split(".")[0] for a in node.names] # type: ignore[attr-defined]
|
|
297
|
+
for top in tops:
|
|
298
|
+
local = (source / top).is_dir() or (source / f"{top}.py").is_file()
|
|
299
|
+
if top in stdlib or local or top in ("py_serverless_kit", "__future__") or top in out:
|
|
300
|
+
continue
|
|
301
|
+
out.append(top)
|
|
302
|
+
return out
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _matching_requirements(root: Path, modules: List[str]) -> Tuple[List[str], List[str]]:
|
|
306
|
+
"""The lines of the project's requirements.txt that name ``modules``, and the modules no line names."""
|
|
307
|
+
path = root / PYTHON_MARKER
|
|
308
|
+
lines = path.read_text("utf-8").splitlines() if path.is_file() else []
|
|
309
|
+
by_name: Dict[str, str] = {}
|
|
310
|
+
for line in lines:
|
|
311
|
+
match = _REQ_NAME.match(line.split("#", 1)[0])
|
|
312
|
+
if match and not line.lstrip().startswith("-"):
|
|
313
|
+
by_name.setdefault(_norm(match.group(1)), line.strip())
|
|
314
|
+
found: List[str] = []
|
|
315
|
+
unknown: List[str] = []
|
|
316
|
+
for module in modules:
|
|
317
|
+
key = _norm(module)
|
|
318
|
+
match_line = by_name.get(key) or by_name.get(_DISTRIBUTIONS.get(key, ""))
|
|
319
|
+
if match_line is None and sys.version_info < (3, 10):
|
|
320
|
+
continue # no stdlib list on 3.9: do not guess
|
|
321
|
+
if match_line:
|
|
322
|
+
line = match_line
|
|
323
|
+
if line not in found:
|
|
324
|
+
found.append(line)
|
|
325
|
+
else:
|
|
326
|
+
unknown.append(module)
|
|
327
|
+
return found, unknown
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def strip_functions(text: str, names: List[str]) -> Tuple[str, List[str]]:
|
|
331
|
+
"""``text`` (a Python module) without the functions ``names`` and the imports only they used: what a
|
|
332
|
+
placement group's copy of a shared code file keeps. A function that code left in the file still calls stays.
|
|
333
|
+
Returns the new text and the names removed."""
|
|
334
|
+
removed: List[str] = []
|
|
335
|
+
for name in names:
|
|
336
|
+
try:
|
|
337
|
+
tree = ast.parse(text)
|
|
338
|
+
except SyntaxError:
|
|
339
|
+
return text, removed
|
|
340
|
+
target = next((n for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
|
|
341
|
+
and n.name == name), None)
|
|
342
|
+
if target is None or _used_elsewhere(tree, target):
|
|
343
|
+
continue
|
|
344
|
+
text = _without(text.splitlines(), tree, target)
|
|
345
|
+
removed.append(name)
|
|
346
|
+
return text, removed
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _used_elsewhere(tree: ast.Module, target: ast.stmt) -> bool:
|
|
350
|
+
name = getattr(target, "name", "")
|
|
351
|
+
return any(name in _uses(node) for node in tree.body if node is not target)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _set_handler(path: Path, function: str, old: str, new: str) -> None:
|
|
355
|
+
"""Point ``functions.<function>.handler`` at ``new``, keeping pskit.yml's comments and layout when the old
|
|
356
|
+
value is written once (else rewrite the file)."""
|
|
357
|
+
text = path.read_text("utf-8")
|
|
358
|
+
pattern = re.compile(r"(^\s*handler:\s*['\"]?)" + re.escape(old) + r"(['\"]?\s*(?:#.*)?$)", re.M)
|
|
359
|
+
if len(pattern.findall(text)) == 1:
|
|
360
|
+
text = pattern.sub(lambda m: m.group(1) + new + m.group(2), text)
|
|
361
|
+
path.write_bytes(text.encode("utf-8"))
|
|
362
|
+
return
|
|
363
|
+
raw = yaml.safe_load(text) or {}
|
|
364
|
+
raw["functions"][function]["handler"] = new
|
|
365
|
+
path.write_bytes(yaml.safe_dump(raw, sort_keys=False, allow_unicode=True, width=120).encode("utf-8"))
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def describe(result: Dict[str, Any]) -> List[str]:
|
|
369
|
+
out = [f"{result['function']}: {result['from']} -> {result['to']} (handler {result['handler']})"]
|
|
370
|
+
if result.get("helpers"):
|
|
371
|
+
out.append(f" copied helpers: {', '.join(result['helpers'])} (still in {result['from']} for the others)")
|
|
372
|
+
if result.get("kept_in_source"):
|
|
373
|
+
out.append(f" kept in {result['from']} too: code there still calls it")
|
|
374
|
+
if result.get("requirements"):
|
|
375
|
+
out.append(f" requirements.txt: {', '.join(result['requirements'])}")
|
|
376
|
+
if result.get("unknown_imports"):
|
|
377
|
+
out.append(f" check requirements.txt for: {', '.join(result['unknown_imports'])}")
|
|
378
|
+
return out
|
|
@@ -63,9 +63,11 @@ def parse(raw: Any, functions: List[str], workflows: Optional[List[str]] = None)
|
|
|
63
63
|
return {}
|
|
64
64
|
if not isinstance(raw, dict):
|
|
65
65
|
raise PlacementError("options.placement must be a mapping with `groups` (and optionally `default`)")
|
|
66
|
-
unknown = sorted(set(raw) - {"default", "groups", "_groups"}) # _groups: resolved
|
|
66
|
+
unknown = sorted(set(raw) - {"default", "groups", "gateway", "_groups"}) # _groups: resolved (resolve())
|
|
67
67
|
if unknown:
|
|
68
|
-
raise PlacementError(f"unknown options.placement keys {unknown}; expected default, groups")
|
|
68
|
+
raise PlacementError(f"unknown options.placement keys {unknown}; expected default, groups, gateway")
|
|
69
|
+
if raw.get("gateway") not in (None, True, False):
|
|
70
|
+
raise PlacementError("options.placement.gateway is true or false")
|
|
69
71
|
groups_raw = raw.get("groups")
|
|
70
72
|
if not isinstance(groups_raw, dict) or not groups_raw:
|
|
71
73
|
raise PlacementError("options.placement.groups must name at least one group: {api: {server: app1}}")
|
|
@@ -130,7 +132,17 @@ def parse(raw: Any, functions: List[str], workflows: Optional[List[str]] = None)
|
|
|
130
132
|
function_group.setdefault(fn, default)
|
|
131
133
|
for name, group in groups.items():
|
|
132
134
|
group["functions"] = sorted(f for f, g in function_group.items() if g == name)
|
|
133
|
-
return {"default": default, "groups": groups, "function_group": function_group
|
|
135
|
+
return {"default": default, "groups": groups, "function_group": function_group,
|
|
136
|
+
**({"gateway": raw["gateway"]} if raw.get("gateway") is not None else {})}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def gateway(parsed: Dict[str, Any], options: Dict[str, Any]) -> bool:
|
|
140
|
+
"""Whether the default group answers every route, passing other groups' routes on (one URL for the project).
|
|
141
|
+
``placement.gateway``; unset: on with a load balancer (it needs one entry point), else off, so each server
|
|
142
|
+
answers only the routes of the functions it runs."""
|
|
143
|
+
if "gateway" in parsed:
|
|
144
|
+
return bool(parsed["gateway"])
|
|
145
|
+
return options.get("balancer") not in (None, "", False)
|
|
134
146
|
|
|
135
147
|
|
|
136
148
|
def group_stage(stage: str, group: str, default: str) -> str:
|
|
@@ -146,6 +158,12 @@ def resource_name(group: str, default: str) -> str:
|
|
|
146
158
|
return "host" if group == default else f"host_{group.replace('-', '_')}"
|
|
147
159
|
|
|
148
160
|
|
|
161
|
+
def group_data_dir(data_dir: str, group: str) -> str:
|
|
162
|
+
"""``<data_dir>/groups/<group>``: a group's queues, buckets and logs, inside the default group's data dir."""
|
|
163
|
+
sep = "\\" if "\\" in data_dir and "/" not in data_dir else "/"
|
|
164
|
+
return data_dir.rstrip("/\\") + f"{sep}groups{sep}{group}"
|
|
165
|
+
|
|
166
|
+
|
|
149
167
|
def group_options(options: Dict[str, Any], group: str, body: Dict[str, Any], default: str) -> Dict[str, Any]:
|
|
150
168
|
"""The provider options one group deploys with: pskit.yml's, without placement, with the group's target."""
|
|
151
169
|
out = copy.deepcopy({k: v for k, v in options.items() if k != "placement"})
|
|
@@ -199,8 +217,10 @@ def resolve(options: Dict[str, Any], root: Path, stage: str, server: Optional[st
|
|
|
199
217
|
else:
|
|
200
218
|
resolve_deploy(opts, root, group_stage(stage, name, default))
|
|
201
219
|
if name != default and opts.get("data_dir"):
|
|
202
|
-
# pskit.yml's data_dir (or the saved server's): each group keeps its own state
|
|
203
|
-
|
|
220
|
+
# pskit.yml's data_dir (or the saved server's): each group keeps its own state in a folder inside it.
|
|
221
|
+
# Never beside it: a saved server's data_dir is often the home folder, and its parent (/home) is not
|
|
222
|
+
# writable.
|
|
223
|
+
opts["data_dir"] = group_data_dir(str(opts["data_dir"]), name)
|
|
204
224
|
resolved[name] = opts
|
|
205
225
|
seen: Dict[Tuple[str, int], str] = {}
|
|
206
226
|
for name, opts in resolved.items():
|