pullup 0.0.6__tar.gz → 0.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pullup-0.0.7/PKG-INFO +168 -0
- pullup-0.0.7/README.md +141 -0
- {pullup-0.0.6 → pullup-0.0.7}/pullup/__init__.py +1 -1
- {pullup-0.0.6 → pullup-0.0.7}/pullup/deploy.py +9 -7
- {pullup-0.0.6 → pullup-0.0.7}/pullup/drive.py +10 -10
- {pullup-0.0.6 → pullup-0.0.7}/pullup/env.py +16 -12
- {pullup-0.0.6 → pullup-0.0.7}/pullup/infra.py +12 -6
- {pullup-0.0.6 → pullup-0.0.7}/pullup/pipeline.py +18 -17
- {pullup-0.0.6 → pullup-0.0.7}/pullup/project.py +11 -11
- {pullup-0.0.6 → pullup-0.0.7}/pullup/stack.py +3 -3
- {pullup-0.0.6 → pullup-0.0.7}/pullup/wfbuild.py +15 -11
- {pullup-0.0.6 → pullup-0.0.7}/pullup/workflows.py +20 -16
- pullup-0.0.6/PKG-INFO +0 -162
- pullup-0.0.6/README.md +0 -135
- {pullup-0.0.6 → pullup-0.0.7}/.gitignore +0 -0
- {pullup-0.0.6 → pullup-0.0.7}/LICENSE +0 -0
- {pullup-0.0.6 → pullup-0.0.7}/pullup/_modidx.py +0 -0
- {pullup-0.0.6 → pullup-0.0.7}/pyproject.toml +0 -0
pullup-0.0.7/PKG-INFO
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pullup
|
|
3
|
+
Version: 0.0.7
|
|
4
|
+
Summary: build, release, and deploy projects
|
|
5
|
+
Project-URL: Repository, https://github.com/vedicreader/pullup
|
|
6
|
+
Project-URL: Documentation, https://vedicreader.github.io/pullup
|
|
7
|
+
Author-email: Karthik Rajgopal <karthik.rajgopal@hotmail.com>
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: jupyter,nbdev,notebook,python
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Natural Language :: English
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: cfeasy>=0.0.4
|
|
18
|
+
Requires-Dist: dockeasy>=0.0.5
|
|
19
|
+
Requires-Dist: fastcore>=1.8.17
|
|
20
|
+
Requires-Dist: gheasy>=0.0.10
|
|
21
|
+
Requires-Dist: keyring>=25.0.0
|
|
22
|
+
Requires-Dist: ptymini>=0.0.5
|
|
23
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
24
|
+
Requires-Dist: pyyaml>=6.0
|
|
25
|
+
Requires-Dist: vpseasy>=0.0.18
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# pullup
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
|
|
32
|
+
|
|
33
|
+
pullup is the engine behind a release and deploy panel. Point it at a project folder and it tells you what kind of project it is (an nbdev library, a plain Python package, a Rust crate and so on) and which commands release it. It then runs those commands in a terminal you can watch, and stops at the first one that fails. The same machinery runs the deploy of a web app to a virtual private server (VPS). pullup also reads the GitHub Actions workflows in the repository, writes new ones, and lists the Hetzner servers and Cloudflare tunnels the project uses.
|
|
34
|
+
|
|
35
|
+
Every public call returns plain dictionaries and lists, so a user interface can draw them and a test can compare them. The work that talks to GitHub, Docker, Hetzner and Cloudflare is done by four sibling libraries: gheasy, dockeasy, vpseasy and cfeasy.
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
``` sh
|
|
40
|
+
pip install pullup
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
This also installs gheasy, dockeasy, vpseasy and cfeasy. None of them is imported until a call needs it, so reading a project or running a release does not load them.
|
|
44
|
+
|
|
45
|
+
## What kind of project is this
|
|
46
|
+
|
|
47
|
+
A release flow is the ordered list of commands that publishes one kind of project. An nbdev library is released with `nbdev_prepare`, `nbdev_bump_version`, `nbdev_release_gh` and `nbdev_pypi`. A plain Python package and a Rust crate each have a different list. [`Project`](https://vedicreader.github.io/pullup/project.html#project) looks at the files in a folder, picks the flow that fits, and returns its steps. The folder here holds only a `pyproject.toml` with a `[tool.nbdev]` table, so it is an nbdev project:
|
|
48
|
+
|
|
49
|
+
``` python
|
|
50
|
+
project = Project(root)
|
|
51
|
+
project.kind, [step.id for step in project.steps()]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
('nbdev', ['prepare', 'bump', 'gh', 'pypi'])
|
|
55
|
+
|
|
56
|
+
## Running the steps
|
|
57
|
+
|
|
58
|
+
A [`Pipeline`](https://vedicreader.github.io/pullup/pipeline.html#pipeline) runs a list of steps for one project. Each step runs in a pseudo-terminal (PTY), so a program that asks for a password behaves as it would in your own shell, and every step’s output goes into one shared transcript. When a step passes, the next one starts by itself. When one fails, the pipeline stops there.
|
|
59
|
+
|
|
60
|
+
`save` stores the plan and `start` launches the first step that has not yet passed. `start` returns as soon as the step is running, not when it finishes, which is why the transcript below shows the command but not yet its output. Call `state` or `tail` whenever you want to see how far it has got.
|
|
61
|
+
|
|
62
|
+
``` python
|
|
63
|
+
pipeline = Pipeline(root, dir='.demo')
|
|
64
|
+
pipeline.save([{'id': 'hello', 'cmd': 'say hello',
|
|
65
|
+
'argv': [sys.executable, '-c', 'print("ready")']}])
|
|
66
|
+
await pipeline.start()
|
|
67
|
+
pipeline.state()['done'], pipeline.tail().splitlines()[-2:]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
(False, ['', '❯ say hello'])
|
|
71
|
+
|
|
72
|
+
## Where plans are kept
|
|
73
|
+
|
|
74
|
+
A saved plan is a JSON file inside the project folder, in the directory you pass as `dir` (`.pullup` unless you say otherwise). pullup never writes a plan outside the project:
|
|
75
|
+
|
|
76
|
+
``` python
|
|
77
|
+
custom = Pipeline(root, dir='.myapp')
|
|
78
|
+
custom.save([{'id': 'show', 'cmd': 'show', 'argv': [sys.executable, '-c', 'print("saved")']}])
|
|
79
|
+
custom.path.relative_to(root)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Path('.myapp/pipeline.json')
|
|
83
|
+
|
|
84
|
+
## Deploying a web app
|
|
85
|
+
|
|
86
|
+
[`Deploy`](https://vedicreader.github.io/pullup/deploy.html#deploy) is a pipeline that puts a web app on a server. Its steps run the subcommands of a `deploy.py` script that lives in the project, and [`Deploy.scaffold`](https://vedicreader.github.io/pullup/deploy.html#deploy.scaffold) can write that script for you. [`Drive`](https://vedicreader.github.io/pullup/drive.html#drive) covers the same ground one library call at a time (write the Dockerfile, open the tunnel, provision the server), so you can run and check each part on its own. Both are pipelines, so they start, stop and report like the one above.
|
|
87
|
+
|
|
88
|
+
``` python
|
|
89
|
+
Deploy(root).state()['kind'], Drive(root).state()['kind']
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
('deploy', 'drive')
|
|
93
|
+
|
|
94
|
+
## Workflows
|
|
95
|
+
|
|
96
|
+
[`Workflows`](https://vedicreader.github.io/pullup/workflows.html#workflows) reads the GitHub Actions files in `.github/workflows` and turns each one into a row of triggers and jobs. Reading local files needs no GitHub token. Listing runs, starting them and pushing secrets do.
|
|
97
|
+
|
|
98
|
+
``` python
|
|
99
|
+
workflow_dir = root/'.github'/'workflows'
|
|
100
|
+
workflow_dir.mkdir(parents=True)
|
|
101
|
+
(workflow_dir/'ci.yml').write_text('''name: CI
|
|
102
|
+
on: push
|
|
103
|
+
jobs:
|
|
104
|
+
test:
|
|
105
|
+
runs-on: ubuntu-latest
|
|
106
|
+
steps:
|
|
107
|
+
- run: pytest
|
|
108
|
+
''')
|
|
109
|
+
workflows = Workflows(root)
|
|
110
|
+
[(row['name'], [job['id'] for job in row['jobs']]) for row in workflows.parsed()]
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
[('CI', ['test'])]
|
|
114
|
+
|
|
115
|
+
## Environment values
|
|
116
|
+
|
|
117
|
+
Many steps need a token or a setting, such as `GITHUB_TOKEN` for a GitHub release. A step lists the names it needs under `needs`, and the pipeline looks them up in an environment store when it starts the step. The values are given to the child process only, and your own process environment is left alone. The default store is an [`EnvStore`](https://vedicreader.github.io/pullup/env.html#envstore), which keeps secrets in the system keychain, but any object with `get` and `values` methods will do, like the small one below:
|
|
118
|
+
|
|
119
|
+
``` python
|
|
120
|
+
class Values:
|
|
121
|
+
def get(self, key, secret=True): return 'demo-token' if key == 'DEMO_TOKEN' else ''
|
|
122
|
+
def values(self, keys): return {key: self.get(key) for key in keys if self.get(key)}
|
|
123
|
+
|
|
124
|
+
with_values = Pipeline(root, env=Values(), dir='.env-demo')
|
|
125
|
+
with_values.save([{'id': 'env', 'cmd': 'read environment', 'needs': ['DEMO_TOKEN'],
|
|
126
|
+
'argv': [sys.executable, '-c', 'import os; print(os.environ["DEMO_TOKEN"])']}])
|
|
127
|
+
await with_values.start()
|
|
128
|
+
with_values.tail().splitlines()[-2:]
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
['', '❯ read environment']
|
|
132
|
+
|
|
133
|
+
## Which package broke
|
|
134
|
+
|
|
135
|
+
When a step fails with a Python traceback, the useful question is which of your own packages raised it. [`blame`](https://vedicreader.github.io/pullup/stack.html#blame) reads the traceback text, finds the deepest frame that belongs to one of the packages named in `family`, and returns that package with the file, the line, the function and the error message:
|
|
136
|
+
|
|
137
|
+
``` python
|
|
138
|
+
trace = '''Traceback (most recent call last):
|
|
139
|
+
File "/tmp/pullup/demo.py", line 7, in release
|
|
140
|
+
raise RuntimeError("stopped")
|
|
141
|
+
RuntimeError: stopped
|
|
142
|
+
'''
|
|
143
|
+
blame(trace, family=['pullup'])
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
{'package': 'pullup',
|
|
147
|
+
'file': '/tmp/pullup/demo.py',
|
|
148
|
+
'line': 7,
|
|
149
|
+
'fn': 'release',
|
|
150
|
+
'error': 'RuntimeError: stopped',
|
|
151
|
+
'open': '/tmp/pullup/demo.py',
|
|
152
|
+
'checkout': '',
|
|
153
|
+
'version': '0.0.4'}
|
|
154
|
+
|
|
155
|
+
## Servers and tunnels
|
|
156
|
+
|
|
157
|
+
[`Infra`](https://vedicreader.github.io/pullup/infra.html#infra) lists and deletes Hetzner servers and Cloudflare tunnels, and lists Cloudflare zones and DNS records, using the tokens in the project’s environment store. `status` says which of the two libraries are installed and which tokens are set, without sending a request. An empty `env` falls back to the default [`EnvStore`](https://vedicreader.github.io/pullup/env.html#envstore), so the answer below reflects the keychain of the machine that ran it:
|
|
158
|
+
|
|
159
|
+
``` python
|
|
160
|
+
infra = Infra(env={})
|
|
161
|
+
infra.status()
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
{'vpseasy': True,
|
|
165
|
+
'cfeasy': True,
|
|
166
|
+
'hcloud_token': True,
|
|
167
|
+
'cf_token': False,
|
|
168
|
+
'keys': ['HCLOUD_TOKEN', 'CLOUDFLARE_API_TOKEN']}
|
pullup-0.0.7/README.md
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# pullup
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
|
|
5
|
+
|
|
6
|
+
pullup is the engine behind a release and deploy panel. Point it at a project folder and it tells you what kind of project it is (an nbdev library, a plain Python package, a Rust crate and so on) and which commands release it. It then runs those commands in a terminal you can watch, and stops at the first one that fails. The same machinery runs the deploy of a web app to a virtual private server (VPS). pullup also reads the GitHub Actions workflows in the repository, writes new ones, and lists the Hetzner servers and Cloudflare tunnels the project uses.
|
|
7
|
+
|
|
8
|
+
Every public call returns plain dictionaries and lists, so a user interface can draw them and a test can compare them. The work that talks to GitHub, Docker, Hetzner and Cloudflare is done by four sibling libraries: gheasy, dockeasy, vpseasy and cfeasy.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
``` sh
|
|
13
|
+
pip install pullup
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
This also installs gheasy, dockeasy, vpseasy and cfeasy. None of them is imported until a call needs it, so reading a project or running a release does not load them.
|
|
17
|
+
|
|
18
|
+
## What kind of project is this
|
|
19
|
+
|
|
20
|
+
A release flow is the ordered list of commands that publishes one kind of project. An nbdev library is released with `nbdev_prepare`, `nbdev_bump_version`, `nbdev_release_gh` and `nbdev_pypi`. A plain Python package and a Rust crate each have a different list. [`Project`](https://vedicreader.github.io/pullup/project.html#project) looks at the files in a folder, picks the flow that fits, and returns its steps. The folder here holds only a `pyproject.toml` with a `[tool.nbdev]` table, so it is an nbdev project:
|
|
21
|
+
|
|
22
|
+
``` python
|
|
23
|
+
project = Project(root)
|
|
24
|
+
project.kind, [step.id for step in project.steps()]
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
('nbdev', ['prepare', 'bump', 'gh', 'pypi'])
|
|
28
|
+
|
|
29
|
+
## Running the steps
|
|
30
|
+
|
|
31
|
+
A [`Pipeline`](https://vedicreader.github.io/pullup/pipeline.html#pipeline) runs a list of steps for one project. Each step runs in a pseudo-terminal (PTY), so a program that asks for a password behaves as it would in your own shell, and every step’s output goes into one shared transcript. When a step passes, the next one starts by itself. When one fails, the pipeline stops there.
|
|
32
|
+
|
|
33
|
+
`save` stores the plan and `start` launches the first step that has not yet passed. `start` returns as soon as the step is running, not when it finishes, which is why the transcript below shows the command but not yet its output. Call `state` or `tail` whenever you want to see how far it has got.
|
|
34
|
+
|
|
35
|
+
``` python
|
|
36
|
+
pipeline = Pipeline(root, dir='.demo')
|
|
37
|
+
pipeline.save([{'id': 'hello', 'cmd': 'say hello',
|
|
38
|
+
'argv': [sys.executable, '-c', 'print("ready")']}])
|
|
39
|
+
await pipeline.start()
|
|
40
|
+
pipeline.state()['done'], pipeline.tail().splitlines()[-2:]
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
(False, ['', '❯ say hello'])
|
|
44
|
+
|
|
45
|
+
## Where plans are kept
|
|
46
|
+
|
|
47
|
+
A saved plan is a JSON file inside the project folder, in the directory you pass as `dir` (`.pullup` unless you say otherwise). pullup never writes a plan outside the project:
|
|
48
|
+
|
|
49
|
+
``` python
|
|
50
|
+
custom = Pipeline(root, dir='.myapp')
|
|
51
|
+
custom.save([{'id': 'show', 'cmd': 'show', 'argv': [sys.executable, '-c', 'print("saved")']}])
|
|
52
|
+
custom.path.relative_to(root)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Path('.myapp/pipeline.json')
|
|
56
|
+
|
|
57
|
+
## Deploying a web app
|
|
58
|
+
|
|
59
|
+
[`Deploy`](https://vedicreader.github.io/pullup/deploy.html#deploy) is a pipeline that puts a web app on a server. Its steps run the subcommands of a `deploy.py` script that lives in the project, and [`Deploy.scaffold`](https://vedicreader.github.io/pullup/deploy.html#deploy.scaffold) can write that script for you. [`Drive`](https://vedicreader.github.io/pullup/drive.html#drive) covers the same ground one library call at a time (write the Dockerfile, open the tunnel, provision the server), so you can run and check each part on its own. Both are pipelines, so they start, stop and report like the one above.
|
|
60
|
+
|
|
61
|
+
``` python
|
|
62
|
+
Deploy(root).state()['kind'], Drive(root).state()['kind']
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
('deploy', 'drive')
|
|
66
|
+
|
|
67
|
+
## Workflows
|
|
68
|
+
|
|
69
|
+
[`Workflows`](https://vedicreader.github.io/pullup/workflows.html#workflows) reads the GitHub Actions files in `.github/workflows` and turns each one into a row of triggers and jobs. Reading local files needs no GitHub token. Listing runs, starting them and pushing secrets do.
|
|
70
|
+
|
|
71
|
+
``` python
|
|
72
|
+
workflow_dir = root/'.github'/'workflows'
|
|
73
|
+
workflow_dir.mkdir(parents=True)
|
|
74
|
+
(workflow_dir/'ci.yml').write_text('''name: CI
|
|
75
|
+
on: push
|
|
76
|
+
jobs:
|
|
77
|
+
test:
|
|
78
|
+
runs-on: ubuntu-latest
|
|
79
|
+
steps:
|
|
80
|
+
- run: pytest
|
|
81
|
+
''')
|
|
82
|
+
workflows = Workflows(root)
|
|
83
|
+
[(row['name'], [job['id'] for job in row['jobs']]) for row in workflows.parsed()]
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
[('CI', ['test'])]
|
|
87
|
+
|
|
88
|
+
## Environment values
|
|
89
|
+
|
|
90
|
+
Many steps need a token or a setting, such as `GITHUB_TOKEN` for a GitHub release. A step lists the names it needs under `needs`, and the pipeline looks them up in an environment store when it starts the step. The values are given to the child process only, and your own process environment is left alone. The default store is an [`EnvStore`](https://vedicreader.github.io/pullup/env.html#envstore), which keeps secrets in the system keychain, but any object with `get` and `values` methods will do, like the small one below:
|
|
91
|
+
|
|
92
|
+
``` python
|
|
93
|
+
class Values:
|
|
94
|
+
def get(self, key, secret=True): return 'demo-token' if key == 'DEMO_TOKEN' else ''
|
|
95
|
+
def values(self, keys): return {key: self.get(key) for key in keys if self.get(key)}
|
|
96
|
+
|
|
97
|
+
with_values = Pipeline(root, env=Values(), dir='.env-demo')
|
|
98
|
+
with_values.save([{'id': 'env', 'cmd': 'read environment', 'needs': ['DEMO_TOKEN'],
|
|
99
|
+
'argv': [sys.executable, '-c', 'import os; print(os.environ["DEMO_TOKEN"])']}])
|
|
100
|
+
await with_values.start()
|
|
101
|
+
with_values.tail().splitlines()[-2:]
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
['', '❯ read environment']
|
|
105
|
+
|
|
106
|
+
## Which package broke
|
|
107
|
+
|
|
108
|
+
When a step fails with a Python traceback, the useful question is which of your own packages raised it. [`blame`](https://vedicreader.github.io/pullup/stack.html#blame) reads the traceback text, finds the deepest frame that belongs to one of the packages named in `family`, and returns that package with the file, the line, the function and the error message:
|
|
109
|
+
|
|
110
|
+
``` python
|
|
111
|
+
trace = '''Traceback (most recent call last):
|
|
112
|
+
File "/tmp/pullup/demo.py", line 7, in release
|
|
113
|
+
raise RuntimeError("stopped")
|
|
114
|
+
RuntimeError: stopped
|
|
115
|
+
'''
|
|
116
|
+
blame(trace, family=['pullup'])
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
{'package': 'pullup',
|
|
120
|
+
'file': '/tmp/pullup/demo.py',
|
|
121
|
+
'line': 7,
|
|
122
|
+
'fn': 'release',
|
|
123
|
+
'error': 'RuntimeError: stopped',
|
|
124
|
+
'open': '/tmp/pullup/demo.py',
|
|
125
|
+
'checkout': '',
|
|
126
|
+
'version': '0.0.4'}
|
|
127
|
+
|
|
128
|
+
## Servers and tunnels
|
|
129
|
+
|
|
130
|
+
[`Infra`](https://vedicreader.github.io/pullup/infra.html#infra) lists and deletes Hetzner servers and Cloudflare tunnels, and lists Cloudflare zones and DNS records, using the tokens in the project’s environment store. `status` says which of the two libraries are installed and which tokens are set, without sending a request. An empty `env` falls back to the default [`EnvStore`](https://vedicreader.github.io/pullup/env.html#envstore), so the answer below reflects the keychain of the machine that ran it:
|
|
131
|
+
|
|
132
|
+
``` python
|
|
133
|
+
infra = Infra(env={})
|
|
134
|
+
infra.status()
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
{'vpseasy': True,
|
|
138
|
+
'cfeasy': True,
|
|
139
|
+
'hcloud_token': True,
|
|
140
|
+
'cf_token': False,
|
|
141
|
+
'keys': ['HCLOUD_TOKEN', 'CLOUDFLARE_API_TOKEN']}
|
|
@@ -29,11 +29,13 @@ DeployError = PipelineError
|
|
|
29
29
|
_generator = 'Leela'
|
|
30
30
|
|
|
31
31
|
def use_generator(name):
|
|
32
|
-
"
|
|
32
|
+
"Set the tool name that generated `setup.py` and `deploy.py` files say wrote them."
|
|
33
33
|
global _generator
|
|
34
34
|
_generator = str(name)
|
|
35
35
|
|
|
36
|
-
def generator():
|
|
36
|
+
def generator():
|
|
37
|
+
"The tool name that generated scripts say wrote them."
|
|
38
|
+
return _generator
|
|
37
39
|
|
|
38
40
|
# %% ../nbs/03_deploy.ipynb #dbcb05a3
|
|
39
41
|
DEPLOY_KEYS = {
|
|
@@ -55,7 +57,7 @@ DEPLOY_STEPS = [
|
|
|
55
57
|
|
|
56
58
|
# %% ../nbs/03_deploy.ipynb #d6728ee1
|
|
57
59
|
def setup_template(app, domain, subdomain='', port=5001, lfs=True):
|
|
58
|
-
"
|
|
60
|
+
"The text of a one-time `setup.py` that prepares a project for deploying the lego way."
|
|
59
61
|
host = f'{subdomain}.{domain}' if subdomain else domain
|
|
60
62
|
patterns = "['*.png', '*.jpg', '*.jpeg', '*.webp', '*.ico', '*.mp3', '*.wav']"
|
|
61
63
|
return f'''"""One-shot setup for {app}: gheasy schema, LFS, .env, the deploy workflow, secrets.
|
|
@@ -164,7 +166,7 @@ if __name__ == '__main__':
|
|
|
164
166
|
|
|
165
167
|
# %% ../nbs/03_deploy.ipynb #695a8160
|
|
166
168
|
def deploy_template(app, domain, subdomain='', port=5001, second_host=''):
|
|
167
|
-
"
|
|
169
|
+
"The text of a `deploy.py` that builds, tunnels and deploys the app; `second_host` adds another hostname."
|
|
168
170
|
host = f'{subdomain}.{domain}' if subdomain else domain
|
|
169
171
|
tunnel = f'{subdomain or app}_{domain}'
|
|
170
172
|
second = f'''
|
|
@@ -303,13 +305,13 @@ if __name__ == '__main__':
|
|
|
303
305
|
|
|
304
306
|
# %% ../nbs/03_deploy.ipynb #d11ca8b9
|
|
305
307
|
class Deploy(Pipeline):
|
|
306
|
-
"The deploy pipeline
|
|
308
|
+
"The deploy pipeline, one step per subcommand of the project's `deploy.py`."
|
|
307
309
|
FILE = DEPLOY_FILE
|
|
308
310
|
KIND = 'deploy'
|
|
309
311
|
@staticmethod
|
|
310
312
|
def defaults(root): return list(DEPLOY_STEPS)
|
|
311
313
|
def target(self):
|
|
312
|
-
"Where this deploy goes
|
|
314
|
+
"Where this deploy goes, which secrets are still unset, and which generated files exist."
|
|
313
315
|
store = self.env if isinstance(self.env, EnvStore) else EnvStore()
|
|
314
316
|
def value(key, default=''): return env_value(store, key, default)
|
|
315
317
|
missing = [k for k, v in DEPLOY_KEYS.items()
|
|
@@ -325,7 +327,7 @@ class Deploy(Pipeline):
|
|
|
325
327
|
'keys': {k: (v is None) for k, v in DEPLOY_KEYS.items()}}
|
|
326
328
|
def state(self): return super().state() | {'target': self.target()}
|
|
327
329
|
def scaffold(self, domain='', subdomain='', app='', force=False, second_host=''):
|
|
328
|
-
"Write
|
|
330
|
+
"Write `deploy.py` and `setup.py`, record the deploy keys, add the deploy workflow, and return the paths written."
|
|
329
331
|
domain = str(domain or '').strip()
|
|
330
332
|
if not domain: raise DeployError('a domain is required — the host this app answers on')
|
|
331
333
|
app = str(app or '').strip() or self.root.name
|
|
@@ -36,15 +36,15 @@ DriveError = PipelineError
|
|
|
36
36
|
# %% ../nbs/04_drive.ipynb #3ab3ed04
|
|
37
37
|
@dataclass
|
|
38
38
|
class Part:
|
|
39
|
-
"One
|
|
39
|
+
"One deploy operation as a code template that can run as a step on its own."
|
|
40
40
|
id: str
|
|
41
|
-
package: str
|
|
41
|
+
package: str # the library the code calls
|
|
42
42
|
label: str
|
|
43
43
|
doc: str
|
|
44
|
-
code: str
|
|
45
|
-
needs: list = field(default_factory=list)
|
|
46
|
-
writes: list = field(default_factory=list)
|
|
47
|
-
produces: list = field(default_factory=list)
|
|
44
|
+
code: str # Python source whose `{placeholders}` are filled from `context`
|
|
45
|
+
needs: list = field(default_factory=list) # keys the code reads
|
|
46
|
+
writes: list = field(default_factory=list) # keys the code stores
|
|
47
|
+
produces: list = field(default_factory=list) # files the code creates
|
|
48
48
|
|
|
49
49
|
# %% ../nbs/04_drive.ipynb #b80f6bff
|
|
50
50
|
PARTS = [
|
|
@@ -111,7 +111,7 @@ PARTS = [
|
|
|
111
111
|
|
|
112
112
|
# %% ../nbs/04_drive.ipynb #0cd71f13
|
|
113
113
|
def context(root, env=None):
|
|
114
|
-
"The values every
|
|
114
|
+
"The project values every part's template is filled with: names, host, tunnel, port and file lists."
|
|
115
115
|
root = Path(root).expanduser().resolve()
|
|
116
116
|
store = env or EnvStore()
|
|
117
117
|
def value(key, default=''): return env_value(store, key, default)
|
|
@@ -151,13 +151,13 @@ def _step(part, exe, ctx):
|
|
|
151
151
|
|
|
152
152
|
# %% ../nbs/04_drive.ipynb #b532d107
|
|
153
153
|
def parts(root, env=None, python=''):
|
|
154
|
-
"Every part as a runnable
|
|
154
|
+
"Every part as a runnable `Step`, its template filled with this project's values."
|
|
155
155
|
ctx = context(root, env)
|
|
156
156
|
return [_step(p, str(python or sys.executable), ctx) for p in PARTS]
|
|
157
157
|
|
|
158
158
|
# %% ../nbs/04_drive.ipynb #bc95dca4
|
|
159
159
|
class Drive(Pipeline):
|
|
160
|
-
"The deploy
|
|
160
|
+
"The deploy as six separate library calls, each a step you can run on its own."
|
|
161
161
|
FILE = DRIVE_FILE
|
|
162
162
|
KIND = 'drive'
|
|
163
163
|
def __init__(self, root, python=None, env=None, dir=DIR):
|
|
@@ -166,7 +166,7 @@ class Drive(Pipeline):
|
|
|
166
166
|
@staticmethod
|
|
167
167
|
def defaults(root): return parts(root)
|
|
168
168
|
def refresh(self):
|
|
169
|
-
"
|
|
169
|
+
"Render the parts again if the store's values changed, unless a plan has been saved."
|
|
170
170
|
if self.path.exists(): return self.steps
|
|
171
171
|
key = json.dumps(context(self.root, self.env), sort_keys=True)
|
|
172
172
|
if key != self._rendered:
|
|
@@ -16,19 +16,19 @@ class EnvError(RuntimeError): pass
|
|
|
16
16
|
|
|
17
17
|
# %% ../nbs/00_env.ipynb #9d27ac2f
|
|
18
18
|
def strip_bundle(env, frozen=None):
|
|
19
|
-
"
|
|
19
|
+
"Remove the frozen-app interpreter variables from `env` when running frozen, and return it."
|
|
20
20
|
if not (getattr(sys, 'frozen', False) if frozen is None else frozen): return env
|
|
21
21
|
for name in BUNDLE_ONLY: env.pop(name, None)
|
|
22
22
|
env['PYTHONUTF8'] = '1'
|
|
23
23
|
return env
|
|
24
24
|
|
|
25
25
|
def clean_env():
|
|
26
|
-
"
|
|
26
|
+
"A copy of this process's environment with the frozen-app interpreter variables removed."
|
|
27
27
|
return strip_bundle(os.environ.copy(), frozen=True)
|
|
28
28
|
|
|
29
29
|
# %% ../nbs/00_env.ipynb #4b56c811
|
|
30
30
|
def venv_env(python=None, env=None):
|
|
31
|
-
"
|
|
31
|
+
"A copy of `env` that runs commands in the virtual environment of `python`, or unchanged when `python` is None."
|
|
32
32
|
env = strip_bundle(dict(os.environ if env is None else env))
|
|
33
33
|
if not python: return env
|
|
34
34
|
bindir = str(Path(python).parent)
|
|
@@ -42,13 +42,17 @@ def venv_env(python=None, env=None):
|
|
|
42
42
|
_extra = 'pullup'
|
|
43
43
|
|
|
44
44
|
def use_extra(name):
|
|
45
|
-
"
|
|
45
|
+
"Set the package name that missing-library errors tell the reader to install."
|
|
46
46
|
global _extra
|
|
47
47
|
_extra = str(name)
|
|
48
48
|
|
|
49
|
-
def extra():
|
|
49
|
+
def extra():
|
|
50
|
+
"The package name that missing-library errors tell the reader to install."
|
|
51
|
+
return _extra
|
|
50
52
|
|
|
51
|
-
def needs_extra(what):
|
|
53
|
+
def needs_extra(what):
|
|
54
|
+
"An error message for `what` that ends with the command that installs the missing library."
|
|
55
|
+
return f'{what}: pip install "{_extra}"'
|
|
52
56
|
|
|
53
57
|
# %% ../nbs/00_env.ipynb #e34073db
|
|
54
58
|
def _dockeasy():
|
|
@@ -76,16 +80,16 @@ def _call(fn, *args, **kwargs):
|
|
|
76
80
|
|
|
77
81
|
# %% ../nbs/00_env.ipynb #9937a32a
|
|
78
82
|
class EnvStore:
|
|
79
|
-
"
|
|
83
|
+
"Read and write one project's tokens and settings through dockeasy's keychain and env file."
|
|
80
84
|
def __init__(self, service='fastops', path=None): self.service, self.path = service, path
|
|
81
85
|
def get(self, key, secret=True):
|
|
82
|
-
"One value
|
|
86
|
+
"One value from the keychain, the environment or the env file, or `''` when none has it."
|
|
83
87
|
env_get, _set, secret_get, _sset = _dockeasy()
|
|
84
88
|
stored = (_call(secret_get, key, service=self.service, path=self.path) if secret
|
|
85
89
|
else _call(env_get, key, path=self.path))
|
|
86
90
|
return stored or os.environ.get(key) or ''
|
|
87
91
|
def set(self, key, value, secret=True):
|
|
88
|
-
"Store one value
|
|
92
|
+
"Store one value: a secret in the keychain and the env file, anything else in the env file only."
|
|
89
93
|
key = str(key or '').strip()
|
|
90
94
|
if not key: raise EnvError('a key is required')
|
|
91
95
|
if not str(value): raise EnvError(f'{key} needs a value')
|
|
@@ -94,7 +98,7 @@ class EnvStore:
|
|
|
94
98
|
else: _call(env_set, key, str(value), path=self.path)
|
|
95
99
|
return {'key': key, 'secret': bool(secret)}
|
|
96
100
|
def unset(self, key):
|
|
97
|
-
"Remove a value from
|
|
101
|
+
"Remove a value from the keychain, the env file and `os.environ`, and say which of the first two held it."
|
|
98
102
|
from dockeasy.core import _FASTOPS_ENV
|
|
99
103
|
from dotenv import unset_key
|
|
100
104
|
key, gone = str(key), []
|
|
@@ -112,7 +116,7 @@ class EnvStore:
|
|
|
112
116
|
os.environ.pop(key, None)
|
|
113
117
|
return {'key': key, 'removed': gone}
|
|
114
118
|
def values(self, keys, secret=True):
|
|
115
|
-
"
|
|
119
|
+
"The keys that have a value, as a dict; keys with no value are left out."
|
|
116
120
|
out = {}
|
|
117
121
|
for k in keys:
|
|
118
122
|
try: v = self.get(k, secret=secret)
|
|
@@ -122,6 +126,6 @@ class EnvStore:
|
|
|
122
126
|
|
|
123
127
|
# %% ../nbs/00_env.ipynb #55636ab5
|
|
124
128
|
def env_value(store, key, default='', secret=False):
|
|
125
|
-
"One value
|
|
129
|
+
"One value from an environment store, or `default` when it is unset or the store raises."
|
|
126
130
|
try: return store.get(key, secret=secret) or default
|
|
127
131
|
except Exception: return default
|
|
@@ -50,14 +50,16 @@ def _why(what, pkg, e):
|
|
|
50
50
|
|
|
51
51
|
# %% ../nbs/08_infra.ipynb #f5433bf9
|
|
52
52
|
class Infra:
|
|
53
|
-
"Hetzner and Cloudflare,
|
|
53
|
+
"Hetzner servers and Cloudflare tunnels, zones and DNS records, using this project's tokens."
|
|
54
54
|
HCLOUD, CF_TOKEN = 'HCLOUD_TOKEN', 'CLOUDFLARE_API_TOKEN'
|
|
55
55
|
def __init__(self, env=None):
|
|
56
56
|
self.env = env or EnvStore()
|
|
57
57
|
def token(self, key):
|
|
58
|
+
"A token from the store, else from this process's environment, else `''`."
|
|
58
59
|
try: return self.env.get(key) or os.environ.get(key, '')
|
|
59
60
|
except Exception: return os.environ.get(key, '')
|
|
60
61
|
def status(self):
|
|
62
|
+
"Which libraries are installed and which tokens are set, without the values."
|
|
61
63
|
return {'vpseasy': installed('vpseasy'), 'cfeasy': installed('cfeasy'),
|
|
62
64
|
'hcloud_token': bool(self.token(self.HCLOUD)),
|
|
63
65
|
'cf_token': bool(self.token(self.CF_TOKEN)),
|
|
@@ -78,27 +80,31 @@ class Infra:
|
|
|
78
80
|
"Every Hetzner server on this token, as vpseasy reports them."
|
|
79
81
|
return list(self._hetzner().servers())
|
|
80
82
|
def delete_server(self, name):
|
|
81
|
-
"Delete one server
|
|
83
|
+
"Delete one server by name; this cannot be undone, so ask the user first."
|
|
82
84
|
name = str(name or '').strip()
|
|
83
85
|
if not name: raise InfraError('name the server to delete')
|
|
84
86
|
known = {s['name'] for s in self.servers()}
|
|
85
87
|
if name not in known: raise InfraError(f'no server called {name}')
|
|
86
88
|
self._hetzner().delete(name)
|
|
87
89
|
return {'deleted': name}
|
|
88
|
-
def keys(self):
|
|
90
|
+
def keys(self):
|
|
91
|
+
"The SSH keys on the Hetzner account."
|
|
92
|
+
return list(self._hetzner().keys())
|
|
89
93
|
def tunnels(self):
|
|
90
|
-
"
|
|
94
|
+
"Cloudflare tunnels that have not been deleted, with their status and connection count."
|
|
91
95
|
rows = [{'id': t.get('id') or '', 'name': t.get('name') or '',
|
|
92
96
|
'status': t.get('status') or '', 'created_at': str(t.get('created_at') or ''),
|
|
93
97
|
'deleted_at': str(t.get('deleted_at') or ''),
|
|
94
98
|
'connections': len(t.get('connections') or [])} for t in self._cf().tunnels()]
|
|
95
99
|
return [r for r in rows if not r['deleted_at']]
|
|
96
100
|
def delete_tunnel(self, tunnel_id):
|
|
101
|
+
"Delete one Cloudflare tunnel by id."
|
|
97
102
|
tunnel_id = str(tunnel_id or '').strip()
|
|
98
103
|
if not tunnel_id: raise InfraError('name the tunnel to delete')
|
|
99
104
|
self._cf().delete_tunnel(tunnel_id)
|
|
100
105
|
return {'deleted': tunnel_id}
|
|
101
106
|
def zones(self):
|
|
107
|
+
"The Cloudflare zones this token can see."
|
|
102
108
|
return [{'id': z.get('id') or '', 'name': z.get('name') or '', 'status': z.get('status') or ''}
|
|
103
109
|
for z in self._cf().zones()]
|
|
104
110
|
def records(self, zone):
|
|
@@ -109,10 +115,10 @@ class Infra:
|
|
|
109
115
|
zid = zone if len(zone) == 32 and '.' not in zone else cf.zone_id(zone)
|
|
110
116
|
return sorted((_record(r) for r in cf.dns_records(zid)), key=lambda r: (r['name'], r['type']))
|
|
111
117
|
def verify(self):
|
|
112
|
-
"cfeasy's
|
|
118
|
+
"Run cfeasy's token check, so a missing permission shows up before a deploy."
|
|
113
119
|
return self._cf().verify()
|
|
114
120
|
def overview(self):
|
|
115
|
-
"
|
|
121
|
+
"Servers, tunnels and zones as `{rows, error}` each, so one failing provider cannot hide the others."
|
|
116
122
|
out = {'status': self.status()}
|
|
117
123
|
for key, fn in (('servers', self.servers), ('tunnels', self.tunnels), ('zones', self.zones)):
|
|
118
124
|
try: out[key] = {'rows': fn(), 'error': ''}
|