pdt-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pdt/__init__.py +6 -0
- pdt/cli.py +177 -0
- pdt/config.py +394 -0
- pdt/deploy.py +106 -0
- pdt/deploy_aws.py +613 -0
- pdt/deploy_aws_fargate.py +449 -0
- pdt/deploy_aws_lambda.py +359 -0
- pdt/deploy_azure.py +531 -0
- pdt/deploy_azure_container_apps.py +328 -0
- pdt/deploy_azure_functions.py +307 -0
- pdt/deploy_common.py +94 -0
- pdt/deploy_google_cloud.py +756 -0
- pdt/deploy_windows.py +376 -0
- pdt/examples/hello-world/config.yml +20 -0
- pdt/examples/hello-world/run.py +41 -0
- pdt/examples/impossible-travel-report/config.yml +48 -0
- pdt/examples/impossible-travel-report/env.template +57 -0
- pdt/examples/impossible-travel-report/run.py +232 -0
- pdt/examples/monday-orphaned-account-report/config.yml +47 -0
- pdt/examples/monday-orphaned-account-report/env.template +57 -0
- pdt/examples/monday-orphaned-account-report/run.py +333 -0
- pdt/gcloud_sdk.py +170 -0
- pdt/scaffold.py +278 -0
- pdt/utils/__init__.py +0 -0
- pdt/utils/email_auth.py +406 -0
- pdt/utils/entra.py +114 -0
- pdt/utils/log.py +34 -0
- pdt/utils/send_email.py +467 -0
- pdt/utils/send_email_graph.py +266 -0
- pdt/utils/web.py +28 -0
- pdt_cli-0.1.0.dist-info/METADATA +325 -0
- pdt_cli-0.1.0.dist-info/RECORD +34 -0
- pdt_cli-0.1.0.dist-info/WHEEL +4 -0
- pdt_cli-0.1.0.dist-info/entry_points.txt +2 -0
pdt/__init__.py
ADDED
pdt/cli.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""pdt — set up, run, and deploy scheduled jobs.
|
|
2
|
+
|
|
3
|
+
Commands:
|
|
4
|
+
init [DIR] create a project here, or in DIR
|
|
5
|
+
examples list the example apps bundled with pdt
|
|
6
|
+
new <app> --from EXAMPLE add an app to the project
|
|
7
|
+
list show every app with its schedule and provider
|
|
8
|
+
validate check config files and required env vars
|
|
9
|
+
run <app> run an app locally
|
|
10
|
+
deploy <app> [--yes] [--profile NAME] deploy an app to its configured platform
|
|
11
|
+
destroy <app> [--yes] [--profile NAME] tear down everything deploy created for an app
|
|
12
|
+
login <app> [--profile NAME] sign in again to the app's cloud provider
|
|
13
|
+
az <args...> run the Azure CLI that pdt installs
|
|
14
|
+
gcloud <args...> run the Google Cloud CLI that pdt installs
|
|
15
|
+
|
|
16
|
+
Every command except `init`, `examples`, `az`, and `gcloud` needs a project.
|
|
17
|
+
pdt finds it by walking up from the working directory to the nearest pdt.yml.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import argparse
|
|
23
|
+
import os
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
from pdt import __version__, config, deploy, scaffold
|
|
29
|
+
from pdt.config import ConfigError
|
|
30
|
+
from pdt.utils.send_email import email_problems, prepare_email_auth
|
|
31
|
+
|
|
32
|
+
CLOUD_CLIS = {"az": "deploy_azure.py", "gcloud": "deploy_google_cloud.py"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def cmd_init(args) -> int:
|
|
36
|
+
return scaffold.init(args.directory, args.yes)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def cmd_examples(_args) -> int:
|
|
40
|
+
return scaffold.list_examples()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def cmd_new(args) -> int:
|
|
44
|
+
return scaffold.new_app(args.app, args.source)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cmd_list(_args) -> int:
|
|
48
|
+
apps = config.find_apps()
|
|
49
|
+
if not apps:
|
|
50
|
+
print("This project has no apps yet.")
|
|
51
|
+
print("Run `pdt examples` to see what you can start from,")
|
|
52
|
+
print("then `pdt new my-report --from <example>`.")
|
|
53
|
+
return 0
|
|
54
|
+
for name in apps:
|
|
55
|
+
try:
|
|
56
|
+
app = config.merged_app(name)
|
|
57
|
+
schedule = app["schedule"] or "-"
|
|
58
|
+
provider = app["platform"].get("provider", "-")
|
|
59
|
+
print(f"{name} schedule={schedule} provider={provider}")
|
|
60
|
+
except ConfigError as e:
|
|
61
|
+
print(f"{name} config error: {e}")
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def cmd_validate(_args) -> int:
|
|
66
|
+
problems = config.validate()
|
|
67
|
+
original_env = os.environ.copy()
|
|
68
|
+
try:
|
|
69
|
+
for name in config.find_apps():
|
|
70
|
+
os.environ.clear()
|
|
71
|
+
os.environ.update(original_env)
|
|
72
|
+
try:
|
|
73
|
+
app = config.merged_app(name)
|
|
74
|
+
except ConfigError:
|
|
75
|
+
continue
|
|
76
|
+
config.load_env(app["dir"])
|
|
77
|
+
for problem in config.check_env(app["env"]):
|
|
78
|
+
problems.append(f"{name}: {problem}")
|
|
79
|
+
if config.uses_email(app):
|
|
80
|
+
for problem in email_problems(app["config"]):
|
|
81
|
+
problems.append(f"{name}: {problem}")
|
|
82
|
+
finally:
|
|
83
|
+
os.environ.clear()
|
|
84
|
+
os.environ.update(original_env)
|
|
85
|
+
if problems:
|
|
86
|
+
for problem in problems:
|
|
87
|
+
print(f"error: {problem}")
|
|
88
|
+
print(f"{len(problems)} problem(s) found.")
|
|
89
|
+
return 1
|
|
90
|
+
print("Configuration is valid.")
|
|
91
|
+
return 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def cmd_run(args) -> int:
|
|
95
|
+
try:
|
|
96
|
+
app = config.merged_app(args.app)
|
|
97
|
+
except ConfigError as e:
|
|
98
|
+
print(f"error: {e}")
|
|
99
|
+
return 1
|
|
100
|
+
if config.uses_email(app):
|
|
101
|
+
env_files = config.load_env(app["dir"])
|
|
102
|
+
problems = email_problems(app["config"], check_oauth=False)
|
|
103
|
+
if problems:
|
|
104
|
+
for problem in problems:
|
|
105
|
+
print(f"error: {args.app}: {problem}")
|
|
106
|
+
return 1
|
|
107
|
+
prepare_email_auth(env_files[0] if env_files else app["dir"] / ".env")
|
|
108
|
+
proc = subprocess.run(["uv", "run", "--script", "run.py"], cwd=app["dir"])
|
|
109
|
+
return proc.returncode
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def cmd_deploy(args) -> int:
|
|
113
|
+
return deploy.deploy(args.app, assume_yes=args.yes, profile=args.profile)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def cmd_login(args) -> int:
|
|
117
|
+
return deploy.login(args.app, profile=args.profile)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def cmd_destroy(args) -> int:
|
|
121
|
+
return deploy.destroy(args.app, assume_yes=args.yes, profile=args.profile)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def main() -> int:
|
|
125
|
+
if len(sys.argv) > 1 and sys.argv[1] in CLOUD_CLIS:
|
|
126
|
+
# Before argparse, so the cloud CLI parses its own flags.
|
|
127
|
+
script = Path(__file__).with_name(CLOUD_CLIS[sys.argv[1]])
|
|
128
|
+
return subprocess.run(
|
|
129
|
+
["uv", "run", "--script", str(script), *sys.argv[1:]]).returncode
|
|
130
|
+
parser = argparse.ArgumentParser(
|
|
131
|
+
prog="pdt", description=__doc__,
|
|
132
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
133
|
+
parser.add_argument("--version", action="version", version=__version__)
|
|
134
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
135
|
+
p = sub.add_parser("init", help="create a project here, or in DIR")
|
|
136
|
+
p.add_argument("directory", nargs="?")
|
|
137
|
+
p.add_argument("--yes", action="store_true", help="accept the defaults, ask nothing")
|
|
138
|
+
p.set_defaults(func=cmd_init)
|
|
139
|
+
sub.add_parser("examples", help="list the bundled example apps").set_defaults(
|
|
140
|
+
func=cmd_examples)
|
|
141
|
+
p = sub.add_parser("new", help="add an app to the project")
|
|
142
|
+
p.add_argument("app")
|
|
143
|
+
p.add_argument("--from", dest="source",
|
|
144
|
+
help="which example to copy; run `pdt examples` to see them")
|
|
145
|
+
p.set_defaults(func=cmd_new)
|
|
146
|
+
sub.add_parser("list", help="show every app").set_defaults(func=cmd_list)
|
|
147
|
+
sub.add_parser("validate", help="check config and env").set_defaults(func=cmd_validate)
|
|
148
|
+
p = sub.add_parser("run", help="run an app locally")
|
|
149
|
+
p.add_argument("app")
|
|
150
|
+
p.set_defaults(func=cmd_run)
|
|
151
|
+
p = sub.add_parser("deploy", help="deploy an app")
|
|
152
|
+
p.add_argument("app")
|
|
153
|
+
p.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
|
|
154
|
+
p.add_argument("--profile", help="AWS profile name (AWS only)")
|
|
155
|
+
p.set_defaults(func=cmd_deploy)
|
|
156
|
+
p = sub.add_parser("login", help="sign in again to an app's cloud provider")
|
|
157
|
+
p.add_argument("app")
|
|
158
|
+
p.add_argument("--profile", help="AWS profile name (AWS only)")
|
|
159
|
+
p.set_defaults(func=cmd_login)
|
|
160
|
+
p = sub.add_parser("destroy", help="tear down an app's deployed resources")
|
|
161
|
+
p.add_argument("app")
|
|
162
|
+
p.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
|
|
163
|
+
p.add_argument("--profile", help="AWS profile name (AWS only)")
|
|
164
|
+
p.set_defaults(func=cmd_destroy)
|
|
165
|
+
args = parser.parse_args()
|
|
166
|
+
try:
|
|
167
|
+
return args.func(args)
|
|
168
|
+
except ConfigError as e:
|
|
169
|
+
print(f"error: {e}")
|
|
170
|
+
return 1
|
|
171
|
+
except KeyboardInterrupt:
|
|
172
|
+
print()
|
|
173
|
+
return 130
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
if __name__ == "__main__":
|
|
177
|
+
sys.exit(main())
|
pdt/config.py
ADDED
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
"""Load, merge, and validate pdt configuration.
|
|
2
|
+
|
|
3
|
+
A project is a directory holding pdt.yml. An app is a directory inside it
|
|
4
|
+
that contains run.py. Commands find the project by walking up from the
|
|
5
|
+
working directory, so pdt works the same whether it was installed from
|
|
6
|
+
PyPI or run from a clone of this repository.
|
|
7
|
+
|
|
8
|
+
Merge order for one app, least to most specific:
|
|
9
|
+
1. `platform:` defaults in the project pdt.yml
|
|
10
|
+
2. the app's entry under `apps:` in pdt.yml
|
|
11
|
+
3. the app's own config.yml
|
|
12
|
+
4. PDT_<APP>_<KEY> environment variables (config keys only)
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
from datetime import date, timedelta
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import yaml
|
|
23
|
+
from dotenv import load_dotenv
|
|
24
|
+
|
|
25
|
+
PROJECT_FILE = "pdt.yml"
|
|
26
|
+
APP_FILE = "config.yml"
|
|
27
|
+
|
|
28
|
+
PROVIDERS = ("google-cloud", "azure", "aws", "snowflake", "windows")
|
|
29
|
+
AWS_RUNTIMES = ("lambda", "fargate")
|
|
30
|
+
AZURE_RUNTIMES = ("functions", "container_apps")
|
|
31
|
+
SCHEDULE_SHORTHAND = {
|
|
32
|
+
"hourly": "0 * * * *",
|
|
33
|
+
"daily": "0 0 * * *",
|
|
34
|
+
"weekly": "0 0 * * 0",
|
|
35
|
+
"monthly": "0 0 1 * *",
|
|
36
|
+
"yearly": "0 0 1 1 *",
|
|
37
|
+
}
|
|
38
|
+
APP_KEYS = {"name", "schedule", "timezone", "platform", "config", "env"}
|
|
39
|
+
PLATFORM_KEYS = {
|
|
40
|
+
"provider", "region", "project",
|
|
41
|
+
"account", "runtime",
|
|
42
|
+
"subscription", "resource_group",
|
|
43
|
+
}
|
|
44
|
+
ENV_KEYS = {"required", "one_of", "optional"}
|
|
45
|
+
CRON_FIELDS = (
|
|
46
|
+
("minute", 0, 59),
|
|
47
|
+
("hour", 0, 23),
|
|
48
|
+
("day-of-month", 1, 31),
|
|
49
|
+
("month", 1, 12),
|
|
50
|
+
("day-of-week", 0, 7),
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class ConfigError(Exception):
|
|
55
|
+
pass
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def load_yaml(path: Path) -> dict:
|
|
59
|
+
if not path.is_file():
|
|
60
|
+
return {}
|
|
61
|
+
try:
|
|
62
|
+
data = yaml.safe_load(path.read_text())
|
|
63
|
+
except yaml.YAMLError as e:
|
|
64
|
+
raise ConfigError(f"{path}: not valid yaml: {e}")
|
|
65
|
+
if data is None:
|
|
66
|
+
return {}
|
|
67
|
+
if not isinstance(data, dict):
|
|
68
|
+
raise ConfigError(f"{path}: expected a mapping at the top level")
|
|
69
|
+
return data
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def find_project(start: Path | None = None) -> Path:
|
|
73
|
+
override = os.environ.get("PDT_PROJECT", "").strip()
|
|
74
|
+
if override != "":
|
|
75
|
+
folder = Path(override).expanduser().resolve()
|
|
76
|
+
if not (folder / PROJECT_FILE).is_file():
|
|
77
|
+
raise ConfigError(f"PDT_PROJECT is {folder}, which has no {PROJECT_FILE}")
|
|
78
|
+
return folder
|
|
79
|
+
folder = (start or Path.cwd()).resolve()
|
|
80
|
+
while True:
|
|
81
|
+
if (folder / PROJECT_FILE).is_file():
|
|
82
|
+
return folder
|
|
83
|
+
if folder.parent == folder:
|
|
84
|
+
raise ConfigError(
|
|
85
|
+
f"this is not a pdt project: no {PROJECT_FILE} here or in any "
|
|
86
|
+
"parent folder. Run `pdt init` to set one up.")
|
|
87
|
+
folder = folder.parent
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def find_apps() -> list[str]:
|
|
91
|
+
names = []
|
|
92
|
+
for child in sorted(find_project().iterdir()):
|
|
93
|
+
if child.name.startswith(".") or not child.is_dir():
|
|
94
|
+
continue
|
|
95
|
+
if (child / "run.py").is_file():
|
|
96
|
+
names.append(child.name)
|
|
97
|
+
return names
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def uses_email(app: dict) -> bool:
|
|
101
|
+
return "pdt.utils.send_email" in (app["dir"] / "run.py").read_text()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def root_app_entry(root_cfg: dict, name: str) -> dict:
|
|
105
|
+
for entry in root_cfg.get("apps") or []:
|
|
106
|
+
if isinstance(entry, dict) and entry.get("name") == name:
|
|
107
|
+
return entry
|
|
108
|
+
return {}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def env_overrides(name: str) -> dict:
|
|
112
|
+
prefix = "PDT_" + name.upper().replace("-", "_") + "_"
|
|
113
|
+
out = {}
|
|
114
|
+
for key, val in os.environ.items():
|
|
115
|
+
if key.startswith(prefix):
|
|
116
|
+
out[key[len(prefix):].lower()] = val
|
|
117
|
+
return out
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def merged_app(name: str) -> dict:
|
|
121
|
+
app_dir = find_project() / name
|
|
122
|
+
if not (app_dir / "run.py").is_file():
|
|
123
|
+
raise ConfigError(f"no app named {name!r} (no {name}/run.py)")
|
|
124
|
+
root_cfg = load_yaml(find_project() / PROJECT_FILE)
|
|
125
|
+
entry = root_app_entry(root_cfg, name)
|
|
126
|
+
own = load_yaml(app_dir / APP_FILE)
|
|
127
|
+
return {
|
|
128
|
+
"name": name,
|
|
129
|
+
"dir": app_dir,
|
|
130
|
+
"schedule": own.get("schedule", entry.get("schedule")),
|
|
131
|
+
"timezone": own.get("timezone", entry.get("timezone", "Etc/UTC")),
|
|
132
|
+
"platform": {
|
|
133
|
+
**(root_cfg.get("platform") or {}),
|
|
134
|
+
**(entry.get("platform") or {}),
|
|
135
|
+
**(own.get("platform") or {}),
|
|
136
|
+
},
|
|
137
|
+
"config": {
|
|
138
|
+
**(entry.get("config") or {}),
|
|
139
|
+
**(own.get("config") or {}),
|
|
140
|
+
**env_overrides(name),
|
|
141
|
+
},
|
|
142
|
+
"env": own.get("env", entry.get("env")) or {},
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def save_platform_key(app: dict, key: str, value: str) -> Path:
|
|
147
|
+
# Edit the text rather than rewrite the yaml, so the user's comments survive.
|
|
148
|
+
project_file = find_project() / PROJECT_FILE
|
|
149
|
+
root_cfg = load_yaml(project_file)
|
|
150
|
+
if (root_cfg.get("platform") or {}).get("provider") == app["platform"].get("provider"):
|
|
151
|
+
path = project_file
|
|
152
|
+
else:
|
|
153
|
+
path = app["dir"] / APP_FILE
|
|
154
|
+
lines = path.read_text().splitlines() if path.is_file() else []
|
|
155
|
+
start = next((i for i, line in enumerate(lines) if line.strip() == "platform:"), None)
|
|
156
|
+
if start is None:
|
|
157
|
+
lines += ["platform:", f' {key}: "{value}"']
|
|
158
|
+
else:
|
|
159
|
+
end = start + 1
|
|
160
|
+
while end < len(lines) and (lines[end].startswith((" ", "\t")) or lines[end].strip() == ""):
|
|
161
|
+
end += 1
|
|
162
|
+
block = range(start + 1, end)
|
|
163
|
+
existing = next((i for i in block if lines[i].strip().startswith(f"{key}:")), None)
|
|
164
|
+
if existing is not None:
|
|
165
|
+
lines[existing] = f'{_indent(lines[existing])}{key}: "{value}"'
|
|
166
|
+
else:
|
|
167
|
+
first = next((i for i in block
|
|
168
|
+
if lines[i].strip() and not lines[i].strip().startswith("#")), None)
|
|
169
|
+
indent = _indent(lines[first]) if first is not None else " "
|
|
170
|
+
lines.insert((first if first is not None else start) + 1, f'{indent}{key}: "{value}"')
|
|
171
|
+
path.write_text("\n".join(lines) + "\n")
|
|
172
|
+
return path
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _indent(line: str) -> str:
|
|
176
|
+
return line[:len(line) - len(line.lstrip())]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def aws_account_problem(account: str) -> str:
|
|
180
|
+
# Deploy reads the account from the credentials and writes it back, so
|
|
181
|
+
# validate only judges a value the user already set.
|
|
182
|
+
account = account.strip()
|
|
183
|
+
if account == "" or (len(account) == 12 and account.isdigit()):
|
|
184
|
+
return ""
|
|
185
|
+
return "That is not an AWS account ID. It is 12 digits, for example 123456789012."
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def cron_expression(schedule) -> str:
|
|
189
|
+
if not isinstance(schedule, str) or schedule.strip() == "":
|
|
190
|
+
raise ConfigError("schedule is missing")
|
|
191
|
+
text = schedule.strip()
|
|
192
|
+
if text.lstrip("@") in SCHEDULE_SHORTHAND:
|
|
193
|
+
return SCHEDULE_SHORTHAND[text.lstrip("@")]
|
|
194
|
+
fields = text.split()
|
|
195
|
+
if len(fields) != 5:
|
|
196
|
+
raise ConfigError(
|
|
197
|
+
f"schedule {schedule!r} is not a shorthand "
|
|
198
|
+
f"({', '.join(SCHEDULE_SHORTHAND)}) or a 5-field cron expression")
|
|
199
|
+
for field, (label, lo, hi) in zip(fields, CRON_FIELDS):
|
|
200
|
+
_cron_values(field, lo, hi, label)
|
|
201
|
+
return text
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _cron_values(field: str, lo: int, hi: int, label: str) -> set[int]:
|
|
205
|
+
values = set()
|
|
206
|
+
for part in field.split(","):
|
|
207
|
+
if part == "":
|
|
208
|
+
raise ConfigError(f"cron {label} field {field!r} contains an empty value")
|
|
209
|
+
span, separator, step_text = part.partition("/")
|
|
210
|
+
if separator and (step_text == "" or "/" in step_text):
|
|
211
|
+
raise ConfigError(f"cron {label} field {field!r} has an invalid step")
|
|
212
|
+
try:
|
|
213
|
+
step = int(step_text) if separator else 1
|
|
214
|
+
except ValueError:
|
|
215
|
+
raise ConfigError(f"cron {label} field {field!r} has a non-numeric step")
|
|
216
|
+
if step < 1:
|
|
217
|
+
raise ConfigError(f"cron {label} field {field!r} has a step below 1")
|
|
218
|
+
if span == "*":
|
|
219
|
+
start, end = lo, hi
|
|
220
|
+
elif "-" in span:
|
|
221
|
+
numbers = span.split("-")
|
|
222
|
+
if len(numbers) != 2:
|
|
223
|
+
raise ConfigError(f"cron {label} field {field!r} has an invalid range")
|
|
224
|
+
try:
|
|
225
|
+
start, end = (int(number) for number in numbers)
|
|
226
|
+
except ValueError:
|
|
227
|
+
raise ConfigError(f"cron {label} field {field!r} has a non-numeric range")
|
|
228
|
+
else:
|
|
229
|
+
try:
|
|
230
|
+
start = int(span)
|
|
231
|
+
except ValueError:
|
|
232
|
+
raise ConfigError(f"cron {label} field {field!r} is not numeric")
|
|
233
|
+
end = hi if separator else start
|
|
234
|
+
if start < lo or end > hi or start > end:
|
|
235
|
+
raise ConfigError(
|
|
236
|
+
f"cron {label} field {field!r} must be between {lo} and {hi}")
|
|
237
|
+
values.update(range(start, end + 1, step))
|
|
238
|
+
return values
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def runs_per_month(cron: str) -> float:
|
|
242
|
+
cron = cron_expression(cron)
|
|
243
|
+
minute, hour, dom, month, dow = cron.split()
|
|
244
|
+
per_day = (len(_cron_values(minute, 0, 59, "minute"))
|
|
245
|
+
* len(_cron_values(hour, 0, 23, "hour")))
|
|
246
|
+
months = _cron_values(month, 1, 12, "month")
|
|
247
|
+
doms = _cron_values(dom, 1, 31, "day-of-month")
|
|
248
|
+
dows = {d % 7 for d in _cron_values(dow, 0, 7, "day-of-week")}
|
|
249
|
+
days = 0
|
|
250
|
+
for offset in range(365): # a representative non-leap year
|
|
251
|
+
day = date(2025, 1, 1) + timedelta(days=offset)
|
|
252
|
+
if day.month not in months:
|
|
253
|
+
continue
|
|
254
|
+
dom_hit = day.day in doms
|
|
255
|
+
dow_hit = (day.weekday() + 1) % 7 in dows
|
|
256
|
+
# cron rule: dom and dow are OR'd only when both are restricted
|
|
257
|
+
if dom == "*" or dow == "*":
|
|
258
|
+
hit = dom_hit and dow_hit
|
|
259
|
+
else:
|
|
260
|
+
hit = dom_hit or dow_hit
|
|
261
|
+
if hit:
|
|
262
|
+
days += 1
|
|
263
|
+
return per_day * days / 12
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def find_env_files(start: Path) -> list[Path]:
|
|
267
|
+
folder = start.resolve()
|
|
268
|
+
files = []
|
|
269
|
+
while True:
|
|
270
|
+
candidate = folder / ".env"
|
|
271
|
+
if candidate.is_file():
|
|
272
|
+
files.append(candidate)
|
|
273
|
+
at_root = (folder / PROJECT_FILE).is_file() or (folder / ".git").exists()
|
|
274
|
+
if at_root or folder.parent == folder:
|
|
275
|
+
break
|
|
276
|
+
folder = folder.parent
|
|
277
|
+
return files
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def load_env(start: Path) -> list[Path]:
|
|
281
|
+
# Closest .env wins; parents only fill keys still empty.
|
|
282
|
+
# Existing process env wins (override=False).
|
|
283
|
+
files = find_env_files(start)
|
|
284
|
+
for path in files:
|
|
285
|
+
load_dotenv(path, override=False)
|
|
286
|
+
load_env_json()
|
|
287
|
+
return files
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def load_env_json() -> None:
|
|
291
|
+
# In the cloud, deploy mounts all app secrets as one json blob.
|
|
292
|
+
raw = os.environ.get("PDT_ENV_JSON", "").strip()
|
|
293
|
+
if raw == "":
|
|
294
|
+
return
|
|
295
|
+
try:
|
|
296
|
+
blob = json.loads(raw)
|
|
297
|
+
except json.JSONDecodeError as e:
|
|
298
|
+
raise ConfigError(f"PDT_ENV_JSON is not valid json: {e}")
|
|
299
|
+
for key, val in blob.items():
|
|
300
|
+
os.environ.setdefault(key, str(val))
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def check_env(env_spec: dict) -> list[str]:
|
|
304
|
+
problems = []
|
|
305
|
+
for name in env_spec.get("required") or []:
|
|
306
|
+
if os.environ.get(name, "").strip() == "":
|
|
307
|
+
problems.append(f"missing required env var {name}")
|
|
308
|
+
groups = env_spec.get("one_of") or []
|
|
309
|
+
if groups:
|
|
310
|
+
satisfied = False
|
|
311
|
+
for group in groups:
|
|
312
|
+
complete = True
|
|
313
|
+
for name in group:
|
|
314
|
+
if os.environ.get(name, "").strip() == "":
|
|
315
|
+
complete = False
|
|
316
|
+
if complete:
|
|
317
|
+
satisfied = True
|
|
318
|
+
if not satisfied:
|
|
319
|
+
choices = " or ".join(" + ".join(group) for group in groups)
|
|
320
|
+
problems.append(f"set one of: {choices}")
|
|
321
|
+
return problems
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def validate() -> list[str]:
|
|
325
|
+
problems = []
|
|
326
|
+
try:
|
|
327
|
+
root_cfg = load_yaml(find_project() / PROJECT_FILE)
|
|
328
|
+
except ConfigError as e:
|
|
329
|
+
return [str(e)]
|
|
330
|
+
for key in root_cfg:
|
|
331
|
+
if key not in {"platform", "apps"}:
|
|
332
|
+
problems.append(f"{PROJECT_FILE}: unknown key {key!r}")
|
|
333
|
+
apps = find_apps()
|
|
334
|
+
for entry in root_cfg.get("apps") or []:
|
|
335
|
+
if not isinstance(entry, dict) or "name" not in entry:
|
|
336
|
+
problems.append(f"{PROJECT_FILE}: every apps entry needs a name")
|
|
337
|
+
continue
|
|
338
|
+
if entry["name"] not in apps:
|
|
339
|
+
problems.append(f"{PROJECT_FILE}: app {entry['name']!r} has no directory with a run.py")
|
|
340
|
+
for key in entry:
|
|
341
|
+
if key not in APP_KEYS:
|
|
342
|
+
problems.append(f"{PROJECT_FILE}: app {entry['name']}: unknown key {key!r}")
|
|
343
|
+
for name in apps:
|
|
344
|
+
problems.extend(validate_app(name))
|
|
345
|
+
return problems
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def validate_app(name: str) -> list[str]:
|
|
349
|
+
problems = []
|
|
350
|
+
where = f"{name}/config.yml"
|
|
351
|
+
try:
|
|
352
|
+
own = load_yaml(find_project() / name / APP_FILE)
|
|
353
|
+
except ConfigError as e:
|
|
354
|
+
return [str(e)]
|
|
355
|
+
for key in own:
|
|
356
|
+
if key == "apps":
|
|
357
|
+
problems.append(f"{where}: the apps list is only allowed in pdt.yml")
|
|
358
|
+
elif key not in APP_KEYS:
|
|
359
|
+
problems.append(f"{where}: unknown key {key!r}")
|
|
360
|
+
own_name = own.get("name")
|
|
361
|
+
if own_name is not None and own_name != name:
|
|
362
|
+
problems.append(f"{where}: name {own_name!r} does not match directory {name!r}")
|
|
363
|
+
try:
|
|
364
|
+
app = merged_app(name)
|
|
365
|
+
except ConfigError as e:
|
|
366
|
+
return problems + [str(e)]
|
|
367
|
+
for key in app["platform"]:
|
|
368
|
+
if key not in PLATFORM_KEYS:
|
|
369
|
+
problems.append(f"{name}: platform: unknown key {key!r}")
|
|
370
|
+
provider = app["platform"].get("provider")
|
|
371
|
+
if provider not in PROVIDERS:
|
|
372
|
+
problems.append(
|
|
373
|
+
f"{name}: platform.provider must be one of: {', '.join(PROVIDERS)}. "
|
|
374
|
+
f"Set it under platform: in {PROJECT_FILE}, or in {name}/{APP_FILE}.")
|
|
375
|
+
if provider == "aws":
|
|
376
|
+
problem = aws_account_problem(str(app["platform"].get("account") or ""))
|
|
377
|
+
if problem != "":
|
|
378
|
+
problems.append(f"{name}: platform.account: {problem}")
|
|
379
|
+
runtime = app["platform"].get("runtime", "lambda")
|
|
380
|
+
if runtime not in AWS_RUNTIMES:
|
|
381
|
+
problems.append(f"{name}: platform.runtime must be one of: {', '.join(AWS_RUNTIMES)}")
|
|
382
|
+
if provider == "azure":
|
|
383
|
+
runtime = app["platform"].get("runtime", "functions")
|
|
384
|
+
if runtime not in AZURE_RUNTIMES:
|
|
385
|
+
problems.append(f"{name}: platform.runtime must be one of: {', '.join(AZURE_RUNTIMES)}")
|
|
386
|
+
if app["schedule"] is not None:
|
|
387
|
+
try:
|
|
388
|
+
cron_expression(app["schedule"])
|
|
389
|
+
except ConfigError as e:
|
|
390
|
+
problems.append(f"{name}: {e}")
|
|
391
|
+
for key in app["env"]:
|
|
392
|
+
if key not in ENV_KEYS:
|
|
393
|
+
problems.append(f"{name}: env: unknown key {key!r}")
|
|
394
|
+
return problems
|
pdt/deploy.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Provider-neutral deploy entry points.
|
|
2
|
+
|
|
3
|
+
Validates the app, then dispatches to one script per provider
|
|
4
|
+
(pdt/deploy_<provider>.py) with `uv run --script`, so each provider
|
|
5
|
+
installs its own SDK packages. Every provider script accepts
|
|
6
|
+
`deploy|destroy|login <app> [--yes] [--profile NAME]`.
|
|
7
|
+
|
|
8
|
+
PDT_PROJECT reaches the provider script through the environment, so the
|
|
9
|
+
child agrees with the parent about which project it is working on.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
import subprocess
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from pdt import config
|
|
19
|
+
from pdt.config import ConfigError
|
|
20
|
+
from pdt.utils.send_email import email_problems, prepare_email_auth
|
|
21
|
+
|
|
22
|
+
PROVIDERS = {
|
|
23
|
+
"google-cloud": "deploy_google_cloud.py",
|
|
24
|
+
"aws": "deploy_aws.py",
|
|
25
|
+
"azure": "deploy_azure.py",
|
|
26
|
+
"windows": "deploy_windows.py",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _load(app_name: str):
|
|
31
|
+
app = config.merged_app(app_name)
|
|
32
|
+
provider = app["platform"].get("provider")
|
|
33
|
+
if provider not in PROVIDERS:
|
|
34
|
+
raise ConfigError(
|
|
35
|
+
f"provider {provider!r} is not supported yet "
|
|
36
|
+
f"(supported: {', '.join(PROVIDERS)})")
|
|
37
|
+
return app, provider
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def dispatch(provider: str, command: str, app_name: str, assume_yes: bool,
|
|
41
|
+
profile: str | None = None) -> int:
|
|
42
|
+
script = Path(__file__).with_name(PROVIDERS[provider])
|
|
43
|
+
args = ["uv", "run", "--script", str(script), command, app_name]
|
|
44
|
+
if assume_yes:
|
|
45
|
+
args.append("--yes")
|
|
46
|
+
if profile:
|
|
47
|
+
args += ["--profile", profile]
|
|
48
|
+
env = dict(os.environ, PDT_PROJECT=str(config.find_project()))
|
|
49
|
+
return subprocess.run(args, check=False, env=env).returncode
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def deploy(app_name: str, assume_yes: bool = False, profile: str | None = None) -> int:
|
|
53
|
+
try:
|
|
54
|
+
app, provider = _load(app_name)
|
|
55
|
+
except ConfigError as e:
|
|
56
|
+
print(f"error: {e}")
|
|
57
|
+
return 1
|
|
58
|
+
problems = config.validate_app(app_name)
|
|
59
|
+
env_files = config.load_env(app["dir"])
|
|
60
|
+
for problem in config.check_env(app["env"]):
|
|
61
|
+
problems.append(f"env: {problem}")
|
|
62
|
+
if app["schedule"] is None:
|
|
63
|
+
problems.append("schedule is required to deploy")
|
|
64
|
+
if config.uses_email(app):
|
|
65
|
+
problems.extend(email_problems(app["config"], check_oauth=False))
|
|
66
|
+
if problems:
|
|
67
|
+
for problem in problems:
|
|
68
|
+
print(f"error: {app_name}: {problem}")
|
|
69
|
+
return 1
|
|
70
|
+
if config.uses_email(app):
|
|
71
|
+
env_file = env_files[0] if env_files else app["dir"] / ".env"
|
|
72
|
+
prepare_email_auth(env_file)
|
|
73
|
+
return dispatch(provider, "deploy", app_name, assume_yes, profile)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def login(app_name: str, profile: str | None = None) -> int:
|
|
77
|
+
try:
|
|
78
|
+
app, provider = _load(app_name)
|
|
79
|
+
except ConfigError as e:
|
|
80
|
+
print(f"error: {e}")
|
|
81
|
+
return 1
|
|
82
|
+
config.load_env(app["dir"])
|
|
83
|
+
return dispatch(provider, "login", app_name, False, profile)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def destroy(app_name: str, assume_yes: bool = False, profile: str | None = None) -> int:
|
|
87
|
+
try:
|
|
88
|
+
app, provider = _load(app_name)
|
|
89
|
+
except ConfigError as e:
|
|
90
|
+
print(f"error: {e}")
|
|
91
|
+
return 1
|
|
92
|
+
config.load_env(app["dir"])
|
|
93
|
+
return dispatch(provider, "destroy", app_name, assume_yes, profile)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def confirm(actions: list[str], assume_yes: bool,
|
|
97
|
+
cost_lines: list[str] | None = None) -> bool:
|
|
98
|
+
print("Plan:")
|
|
99
|
+
for action in actions:
|
|
100
|
+
print(f" {action}")
|
|
101
|
+
for line in cost_lines or []:
|
|
102
|
+
print(line)
|
|
103
|
+
if assume_yes:
|
|
104
|
+
return True
|
|
105
|
+
answer = input("Proceed? [y/N] ").strip().lower()
|
|
106
|
+
return answer in ("y", "yes")
|