wos-deploy 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wos_deploy/__init__.py +3 -0
- wos_deploy/cli.py +44 -0
- wos_deploy/templates/README.md +252 -0
- wos_deploy/templates/docker-compose.rfq-feeder-primary.yml +15 -0
- wos_deploy/templates/docker-compose.rfq-feeder-secondary.yml +9 -0
- wos_deploy/templates/docker-compose.rfq.yml +19 -0
- wos_deploy/templates/docker-compose.secondary.yml +36 -0
- wos_deploy/templates/docker-compose.yml +138 -0
- wos_deploy/templates/scripts/__init__.py +1 -0
- wos_deploy/templates/scripts/functions/__init__.py +1 -0
- wos_deploy/templates/scripts/functions/common/__init__.py +1 -0
- wos_deploy/templates/scripts/functions/common/setup_database.py +170 -0
- wos_deploy/templates/scripts/functions/common/setup_executable_hosts.py +182 -0
- wos_deploy/templates/scripts/functions/common/setup_namespace.py +22 -0
- wos_deploy/templates/scripts/functions/common/ssh_self_trust.py +119 -0
- wos_deploy/templates/scripts/functions/global/010_core_setup_namespace.py +6 -0
- wos_deploy/templates/scripts/functions/global/020_core_setup_master.py +46 -0
- wos_deploy/templates/scripts/functions/global/021_setup_database.py +5 -0
- wos_deploy/templates/scripts/functions/global/042_setup_global_ctp.py +67 -0
- wos_deploy/templates/scripts/functions/global/099_core_setup_runtime_scripts.py +85 -0
- wos_deploy/templates/scripts/functions/global/__init__.py +1 -0
- wos_deploy/templates/scripts/functions/private/010_core_setup_namespace.py +6 -0
- wos_deploy/templates/scripts/functions/private/021_setup_database.py +5 -0
- wos_deploy/templates/scripts/functions/private/030_core_setup_gateway.py +15 -0
- wos_deploy/templates/scripts/functions/private/040_core_setup_executable_hosts.py +6 -0
- wos_deploy/templates/scripts/functions/private/099_core_setup_runtime_scripts.py +73 -0
- wos_deploy/templates/scripts/functions/private/100_rfq_setup_addon.py +201 -0
- wos_deploy/templates/scripts/functions/private/__init__.py +1 -0
- wos_deploy/templates/scripts/functions/run_function.py +56 -0
- wos_deploy/templates/scripts/functions/utils.py +43 -0
- wos_deploy/templates/scripts/init/chown.sh +8 -0
- wos_deploy/templates/scripts/init/postinit.sh +50 -0
- wos_deploy/templates/scripts/init/preinit.sh +5 -0
- wos_deploy/templates/templates/setup_database.yml.template +21 -0
- wos_deploy/templates/templates/setup_gateway.yml.template +1 -0
- wos_deploy/templates/templates/setup_global_ctp.yml.template +8 -0
- wos_deploy/templates/wos-deploy.py +1779 -0
- wos_deploy-2.1.0.dist-info/METADATA +95 -0
- wos_deploy-2.1.0.dist-info/RECORD +43 -0
- wos_deploy-2.1.0.dist-info/WHEEL +5 -0
- wos_deploy-2.1.0.dist-info/entry_points.txt +3 -0
- wos_deploy-2.1.0.dist-info/licenses/LICENSE +202 -0
- wos_deploy-2.1.0.dist-info/top_level.txt +1 -0
wos_deploy/__init__.py
ADDED
wos_deploy/cli.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
WOS Deployment Manager - pip-installable CLI entry point.
|
|
4
|
+
|
|
5
|
+
Thin wrapper that imports the deployment logic and patches the template
|
|
6
|
+
directory to point at the bundled package data instead of the script's
|
|
7
|
+
parent directory.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import importlib.util
|
|
11
|
+
import sys
|
|
12
|
+
from importlib.resources import files as pkg_files
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main():
|
|
17
|
+
# Resolve the bundled templates directory
|
|
18
|
+
templates_dir = Path(str(pkg_files("wos_deploy") / "templates"))
|
|
19
|
+
|
|
20
|
+
# Load wos-deploy.py as a proper module so @dataclass etc. work
|
|
21
|
+
module_name = "wos_deploy_impl"
|
|
22
|
+
spec = importlib.util.spec_from_file_location(
|
|
23
|
+
module_name,
|
|
24
|
+
templates_dir / "wos-deploy.py",
|
|
25
|
+
)
|
|
26
|
+
mod = importlib.util.module_from_spec(spec)
|
|
27
|
+
sys.modules[module_name] = mod
|
|
28
|
+
spec.loader.exec_module(mod)
|
|
29
|
+
|
|
30
|
+
# Patch WOSDeployment so script_dir resolves to our bundled templates
|
|
31
|
+
_original_init = mod.WOSDeployment.__init__
|
|
32
|
+
|
|
33
|
+
def _patched_init(self):
|
|
34
|
+
_original_init(self)
|
|
35
|
+
self.script_dir = templates_dir
|
|
36
|
+
|
|
37
|
+
mod.WOSDeployment.__init__ = _patched_init
|
|
38
|
+
|
|
39
|
+
# Run
|
|
40
|
+
mod.main()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
main()
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
# WOS Deployment
|
|
2
|
+
|
|
3
|
+
Docker-based deployment tooling for WOS instances.
|
|
4
|
+
|
|
5
|
+
## Requirements
|
|
6
|
+
|
|
7
|
+
- Python 3.8+
|
|
8
|
+
- Docker Engine with `docker compose`
|
|
9
|
+
- Linux host on `x64` or `arm64`
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
The command is `wos-deploy` (`wos` is kept as an alias; a deployment folder created by `setup-folder` from a source checkout also carries a `./wos` wrapper).
|
|
14
|
+
|
|
15
|
+
Create an instance:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
wos-deploy setup-folder ../wos-prod --env prod --port-offset 3
|
|
19
|
+
cd ../wos-prod
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Resolve topology and run initialization:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
wos-deploy setup-private --wosgw <global-gateway-host>:<port>
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
`--wosgw` (or `WOSGW` in the environment) is required: there is no built-in gateway.
|
|
29
|
+
|
|
30
|
+
or:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
wos-deploy setup-global --enable_ctp --enable_secondary
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Start services:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
wos-deploy deploy
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## CLI
|
|
43
|
+
|
|
44
|
+
- `setup-folder <folder_name> [--template-dir PATH] [--env dev|prod] [--port-offset N]`
|
|
45
|
+
- `setup-private [-r|--run] [--enable_ctp] [--enable_secondary] --wosgw HOST:PORT`
|
|
46
|
+
- `setup-global [-r|--run] [--enable_ctp] [--enable_secondary]`
|
|
47
|
+
- `setup-rfq [-r] [--render-only] [--registry R] --image-tag T [--enable_ctp] [--enable_secondary] [...]` (see below)
|
|
48
|
+
- `--version`
|
|
49
|
+
- `deploy [--skip-pull]`
|
|
50
|
+
- `restart [service]`
|
|
51
|
+
- `shutdown [service]`
|
|
52
|
+
- `status [-j|--json]`
|
|
53
|
+
- `logs [-f|--follow] [service]`
|
|
54
|
+
- `shell [-u|--user USER] [--shell PATH] service`
|
|
55
|
+
- `reset`
|
|
56
|
+
- `config`
|
|
57
|
+
|
|
58
|
+
## Flow
|
|
59
|
+
|
|
60
|
+
`setup-folder` creates the instance directory, copies `templates/`, symlinks the deployment assets, and creates the base `volumes/` layout.
|
|
61
|
+
|
|
62
|
+
The deployment uses:
|
|
63
|
+
|
|
64
|
+
- `docker-compose.yml` for `wos-init` and `wos-primary`
|
|
65
|
+
- `docker-compose.secondary.yml` for optional `wos-secondary`
|
|
66
|
+
- `.config` as a small env-style state file written by `setup-folder`
|
|
67
|
+
- `.env` as the runtime variable file consumed by compose and init/functions
|
|
68
|
+
|
|
69
|
+
`setup-private` and `setup-global` now do the real initialization work:
|
|
70
|
+
|
|
71
|
+
1. read `.config`
|
|
72
|
+
2. show the resolved configuration and ask for confirmation unless `-r` is used
|
|
73
|
+
3. resolve topology and write `.env`
|
|
74
|
+
4. copy enabled `templates/<namespace>/*.yml` files into `volumes/data/init/<namespace>/`
|
|
75
|
+
5. pull the required images
|
|
76
|
+
6. generate runtime scripts in `volumes/data/` and `volumes/data-secondary/` through `099_core_setup_runtime_scripts.py`
|
|
77
|
+
7. start a temporary `wos-init` container
|
|
78
|
+
8. execute [`preinit.sh`](scripts/init/preinit.sh)
|
|
79
|
+
9. execute `/usr/bin/entrypoint.sh`
|
|
80
|
+
10. stop transient processes that are not needed after init
|
|
81
|
+
11. execute [`postinit.sh`](scripts/init/postinit.sh)
|
|
82
|
+
12. gracefully flush and stop MySQL in the init container so writes under `volumes/data/mysql/` are persisted
|
|
83
|
+
13. remove the temporary init container
|
|
84
|
+
|
|
85
|
+
`deploy` only starts services. It does not run initialization anymore.
|
|
86
|
+
When private secondary is enabled, `deploy` and `restart` also refresh `known_hosts` for the secondary host inside `wos-primary`.
|
|
87
|
+
|
|
88
|
+
## Runtime Layout
|
|
89
|
+
|
|
90
|
+
```text
|
|
91
|
+
instance/
|
|
92
|
+
├── .config
|
|
93
|
+
├── .env
|
|
94
|
+
├── templates/
|
|
95
|
+
└── volumes/
|
|
96
|
+
├── data/
|
|
97
|
+
│ ├── conf/
|
|
98
|
+
│ ├── mysql/
|
|
99
|
+
│ ├── logs/
|
|
100
|
+
│ ├── backtests/
|
|
101
|
+
│ ├── CA/
|
|
102
|
+
│ ├── init/
|
|
103
|
+
│ │ ├── private/
|
|
104
|
+
│ │ └── global/
|
|
105
|
+
│ ├── start.sh
|
|
106
|
+
│ ├── start-secondary.sh
|
|
107
|
+
│ ├── healthcheck.sh
|
|
108
|
+
│ └── healthcheck-secondary.sh
|
|
109
|
+
├── data-secondary/
|
|
110
|
+
└── ssh-shared/
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The primary and init containers mount the full instance workspace at `/workspace`, so function hooks read:
|
|
114
|
+
|
|
115
|
+
- `.env` from `/workspace/.env`
|
|
116
|
+
- init templates from `/workspace/volumes/data/init/<namespace>/`
|
|
117
|
+
|
|
118
|
+
`wos-primary` and `wos-secondary` keep their environment surface minimal. Most derived values stay in `.env` for compose expansion and init-time function hooks rather than being injected into the runtime containers.
|
|
119
|
+
|
|
120
|
+
## Init Hooks
|
|
121
|
+
|
|
122
|
+
`postinit.sh`:
|
|
123
|
+
|
|
124
|
+
1. ensures `~/.my.cnf` exists
|
|
125
|
+
2. creates SSH keys if missing
|
|
126
|
+
3. scans `scripts/functions/<namespace>/*.py` in filename order
|
|
127
|
+
4. executes only files with a numeric prefix such as `010_...`
|
|
128
|
+
5. always executes scripts whose logical name contains `core`
|
|
129
|
+
6. executes non-`core` scripts only when `volumes/data/init/<namespace>/<logical_name>.yml` exists
|
|
130
|
+
|
|
131
|
+
Function modules are resolved by [`run_function.py`](scripts/functions/run_function.py). If `volumes/data/init/<namespace>/.state/<name>.done` exists, that step is skipped.
|
|
132
|
+
|
|
133
|
+
## Feature Support
|
|
134
|
+
|
|
135
|
+
The following features are supported during the `setup-private` or `setup-global` initialization phase:
|
|
136
|
+
|
|
137
|
+
| Feature | Core | Description |
|
|
138
|
+
| :--- | :---: | :--- |
|
|
139
|
+
| `setup_namespace` | Yes | Cleans up conflicting namespace data from index tables. |
|
|
140
|
+
| `setup_master` | Yes | Generates the `master.yml` configuration for the instance. |
|
|
141
|
+
| `setup_gateway` | Yes | Configures the WOS gateway connection using the `WOSGW` environment variable. |
|
|
142
|
+
| `setup_executable_hosts` | Yes | Registers the primary (and secondary) host architecture and SSH keys in the database. |
|
|
143
|
+
| `setup_database` | No | Configures Rails `database.yml`, syncs `Previlige`/`Dirac` credentials, and manages `database_pools`. |
|
|
144
|
+
| `setup_global_ctp` | No | Configures CTP client and Dirac web service INI files (Global namespace only). |
|
|
145
|
+
|
|
146
|
+
"Core" features always run. Other features run only if a corresponding `.yml` configuration file exists in `templates/<namespace>/`.
|
|
147
|
+
|
|
148
|
+
## Templates
|
|
149
|
+
|
|
150
|
+
Built-in `.template` files are shipped under `templates/` as reference material.
|
|
151
|
+
|
|
152
|
+
`.template` files are reference files only. They are not executed and are not copied into the runtime init directory automatically.
|
|
153
|
+
|
|
154
|
+
To enable an optional function, create the matching `.yml` file under:
|
|
155
|
+
|
|
156
|
+
- `templates/private/`
|
|
157
|
+
- `templates/global/`
|
|
158
|
+
|
|
159
|
+
Only `templates/<namespace>/*.yml` files are copied to `volumes/data/init/<namespace>/` and executed by `postinit.sh`.
|
|
160
|
+
|
|
161
|
+
`--wosgw` is written to `.env` as `WOSGW`, and the private gateway setup step reads it directly from there.
|
|
162
|
+
|
|
163
|
+
## Topology Rules
|
|
164
|
+
|
|
165
|
+
- `CTP` is only supported on `x64` images.
|
|
166
|
+
- On an `arm64` host, `--enable_ctp` requires `--enable_secondary`.
|
|
167
|
+
- On an `x64` host, CTP runs on primary unless secondary is enabled.
|
|
168
|
+
- Without `--enable_ctp`, secondary is disabled unless explicitly requested.
|
|
169
|
+
|
|
170
|
+
## Ports
|
|
171
|
+
|
|
172
|
+
Ports are derived from `--port-offset`:
|
|
173
|
+
|
|
174
|
+
- primary SSH: `22334 + offset`
|
|
175
|
+
- secondary SSH: `22434 + offset`
|
|
176
|
+
- MySQL: `3306 + offset`
|
|
177
|
+
- HTTP: `8000 + offset`
|
|
178
|
+
- HTTPS: `4430 + offset`
|
|
179
|
+
- WOS app range: `((3 + offset) * 1000 + 100)` through `+105`
|
|
180
|
+
|
|
181
|
+
With the default offset `3`:
|
|
182
|
+
|
|
183
|
+
- primary SSH `22337`
|
|
184
|
+
- secondary SSH `22437`
|
|
185
|
+
- MySQL `3309`
|
|
186
|
+
- HTTP `8003`
|
|
187
|
+
- HTTPS `4433`
|
|
188
|
+
- WOS port range `6100-6105`
|
|
189
|
+
|
|
190
|
+
## The tick-quote add-on kind (`setup-rfq`)
|
|
191
|
+
|
|
192
|
+
`setup-rfq` configures a deployment folder as a **tick-quote add-on (RFQ) instance**:
|
|
193
|
+
the storage ring (web backend, soldiers, storage master) + the CSV importer + its upload
|
|
194
|
+
web on `/rfq/` — no time machine, dirac, samplers or privilege. Optionally the realtime
|
|
195
|
+
CTP feeder runs on an x64 service with **your** CTP account.
|
|
196
|
+
|
|
197
|
+
It uses the same machinery as `setup-private` (topology, `.env`, the transient `wos-init`
|
|
198
|
+
cycle). The member set is the server image's own start role `WOS_ROLE=rfq`, so the image
|
|
199
|
+
must carry it: images of the regular `wos-prod-s` family built from a source that has the
|
|
200
|
+
role (older tags, including today's `:latest` on Docker Hub, do not). `setup-rfq` checks the
|
|
201
|
+
image before initialising and stops with a clear message if the role is missing.
|
|
202
|
+
`--image-tag` is required: there is no default tag, so the tool only ever pulls the tag you
|
|
203
|
+
name (without it `setup-rfq` stops and names `--image-tag` and `--registry`).
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
wos-deploy setup-folder ./my-rfq --env prod --port-offset 37
|
|
207
|
+
cd my-rfq
|
|
208
|
+
wos-deploy setup-rfq -r --image-tag <tag-with-the-role> # import-only: no credential asked
|
|
209
|
+
wos-deploy deploy
|
|
210
|
+
wos-deploy config # RFQ_MASTER=localhost:<(3+offset)*1000+100> RFQ_UPLOAD_WEB=http://localhost:<8000+offset>/rfq/api/
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Realtime (feeder on an x64 service; on an arm64 host it runs emulated as the secondary):
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
export WOS_RFQ_CTP_PASSWORD=... WOS_RFQ_MARKET_PASSWORD=... # or answer the prompts (no echo)
|
|
217
|
+
wos-deploy setup-rfq -r --enable_ctp --enable_secondary --image-tag <tag> \
|
|
218
|
+
--ctp-broker-id <broker> --ctp-front tcp://<front-host>:<port> --ctp-investor-id <investor> \
|
|
219
|
+
--market-url https://<full ring>/private-api --market-account <account> --dirac-client-uuid <uuid>
|
|
220
|
+
wos-deploy deploy
|
|
221
|
+
wos-deploy logs wos-secondary # [wos-rfq] lines: ini (masked), front reachability, master link, CTP login verdict
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
| Option | Meaning |
|
|
225
|
+
|---|---|
|
|
226
|
+
| `--registry R` | replaces the Docker Hub namespace `glacierx` in `R/wos-prod-s[-arm64]:<tag>` (e.g. a private `host:5000`); recorded as `IMAGE_REGISTRY` |
|
|
227
|
+
| `--image-tag T` | **required** image tag (no default; only the named tag is pulled); recorded as `IMAGE_TAG`. `WOS_PRIMARY_IMAGE`/`WOS_SECONDARY_IMAGE` still override the whole reference |
|
|
228
|
+
| `--enable_ctp` / `--enable_secondary` | the realtime feeder; same topology rules as `setup-private` (arm64 host: both flags; x64 host: on the primary unless `--enable_secondary`) |
|
|
229
|
+
| `--meta-upstream HOST:PORT` | master serving `global::Market/Futures/...` for uploads with `lead_only=1` (default the own master: `lead_only=0` only) |
|
|
230
|
+
| `--upload-max-bytes N` | upload limit (default 1 GiB) |
|
|
231
|
+
| `--render-only [--host-arch arm64\|x64]` | write `.env` + overrides and validate `docker compose config`; no pull, no init (works without docker for the render part) |
|
|
232
|
+
|
|
233
|
+
Realtime inputs, each from a flag, an environment variable of the name shown, or a prompt
|
|
234
|
+
(passwords: environment or prompt only): `WOS_RFQ_CTP_BROKER_ID`, `WOS_RFQ_CTP_FRONT_END`,
|
|
235
|
+
`WOS_RFQ_CTP_INVESTOR_ID`, `WOS_RFQ_CTP_PASSWORD`, `WOS_RFQ_MARKET_URL`,
|
|
236
|
+
`WOS_RFQ_MARKET_ACCOUNT`, `WOS_RFQ_MARKET_PASSWORD`, `WOS_RFQ_DIRAC_CLIENT_UUID`. They are
|
|
237
|
+
stored only in `./rfq-ctp.env` (mode 0600), mounted read-only into the feeder service and
|
|
238
|
+
rendered there into a 0600 ini; never in `.env`, the compose config or a log. `wos-deploy config`
|
|
239
|
+
shows the two endpoint addresses and prints everything else as `***`. Re-running
|
|
240
|
+
`setup-rfq` re-renders everything and keeps stored inputs unless re-entered.
|
|
241
|
+
|
|
242
|
+
What the developer must bring (the tool cannot provide it):
|
|
243
|
+
|
|
244
|
+
| Needed | Without it |
|
|
245
|
+
|---|---|
|
|
246
|
+
| an image tag carrying `WOS_ROLE=rfq` | `setup-rfq: <image> has no WOS_ROLE=rfq start role ...` and nothing is initialised |
|
|
247
|
+
| (realtime) a CTP account: broker id, investor id, password, a market-data front | front unreachable: `[wos-rfq] ERROR: CTP front ... is NOT reachable`, then `ERROR: no CTP login after 60 s and no answer from the front`; refused login: `ERROR: CTP login REFUSED ... OnRspUserLogin <code> (<msg>)` |
|
|
248
|
+
| (realtime) an account + a client uuid on a full ring's web API (the market list) | the feeder log shows `query_markets failed status=...` and subscribes nothing |
|
|
249
|
+
| (realtime) an x64 host, or amd64 emulation on arm64 | the feeder binary exists only in the amd64 image |
|
|
250
|
+
| (realtime) an image whose feeder reconnects to a restarted master | after `wos-deploy restart wos-primary` an older feeder stays disconnected: restart `wos-secondary` too (the tool warns) |
|
|
251
|
+
|
|
252
|
+
Data lives under `volumes/` (`volumes/data/mysql`, `volumes/data/rfq/upload`); `wos-deploy reset` wipes it.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# RFQ kind, realtime enabled, feeder on the primary (an x64 host without a secondary).
|
|
2
|
+
# The launcher (conf/rfq-feeder-launcher.sh, written by 100_rfq_setup_addon.py) needs
|
|
3
|
+
# root to read the 0600 secrets file and render the feeder ini; start.sh runs as the ring
|
|
4
|
+
# user, so the launcher is started here, in the background, before the image entrypoint.
|
|
5
|
+
services:
|
|
6
|
+
wos-primary:
|
|
7
|
+
volumes:
|
|
8
|
+
- ./rfq-ctp.env:/run/wos-rfq/ctp.env:ro
|
|
9
|
+
- ./volumes/data/rfq/ctp:/home/wolverine/rfq-ctp
|
|
10
|
+
command:
|
|
11
|
+
[
|
|
12
|
+
"/bin/sh",
|
|
13
|
+
"-c",
|
|
14
|
+
"chmod +x /home/wolverine/start.sh /home/wolverine/healthcheck.sh || true; if [ -s /home/wolverine/requirements.txt ]; then /home/wolverine/.pyenv/python3/bin/pip install -r /home/wolverine/requirements.txt; fi; (bash /home/wolverine/conf/rfq-feeder-launcher.sh > /dev/null 2>&1 &); /usr/bin/entrypoint.sh; tail -f /dev/null",
|
|
15
|
+
]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# RFQ kind, realtime enabled, feeder on the x64 secondary (an arm64 host, or an x64 host
|
|
2
|
+
# with --enable_secondary). The secondary's start.sh is the feeder launcher written by
|
|
3
|
+
# 100_rfq_setup_addon.py; it reads the developer's CTP / market-list settings from the
|
|
4
|
+
# deployment's 0600 secrets file mounted read-only below (never from .env, never baked).
|
|
5
|
+
services:
|
|
6
|
+
wos-secondary:
|
|
7
|
+
volumes:
|
|
8
|
+
- ./rfq-ctp.env:/run/wos-rfq/ctp.env:ro
|
|
9
|
+
- ./volumes/data-secondary/rfq-ctp:/home/wolverine/rfq-ctp
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# The tick-quote add-on (RFQ) deployment kind -- `wos setup-rfq` -- layered on
|
|
2
|
+
# docker-compose.yml. The member set comes from the server image's own start role
|
|
3
|
+
# (WOS_ROLE=rfq): the image's setup, run once in the transient wos-init container,
|
|
4
|
+
# generates the add-on's start.sh (storage ring + CSV importer + upload web; no time
|
|
5
|
+
# machine, dirac, samplers or privilege), the importer's ini and the /rfq/ nginx
|
|
6
|
+
# location. WOS_ROLE is an init selector like WOS_BACKEND_IMPL: read once, when the
|
|
7
|
+
# generated layer is written. The primary only needs the importer's upload/state
|
|
8
|
+
# directory on the persistent volumes/ tree (wiped by `wos reset`).
|
|
9
|
+
services:
|
|
10
|
+
wos-init:
|
|
11
|
+
environment:
|
|
12
|
+
- WOS_ROLE=rfq
|
|
13
|
+
- WOS_RFQ_META_UPSTREAM=${WOS_RFQ_META_UPSTREAM:-127.0.0.1:3100}
|
|
14
|
+
- WOS_RFQ_UPLOAD_MAX_BYTES=${WOS_RFQ_UPLOAD_MAX_BYTES:-1073741824}
|
|
15
|
+
volumes:
|
|
16
|
+
- ./volumes/data/rfq/upload:/home/wolverine/backfilling_data
|
|
17
|
+
wos-primary:
|
|
18
|
+
volumes:
|
|
19
|
+
- ./volumes/data/rfq/upload:/home/wolverine/backfilling_data
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
services:
|
|
2
|
+
wos-secondary:
|
|
3
|
+
image: ${SECONDARY_IMAGE}
|
|
4
|
+
platform: ${SECONDARY_PLATFORM}
|
|
5
|
+
container_name: ${SECONDARY_HOSTNAME}
|
|
6
|
+
hostname: ${SECONDARY_HOSTNAME}
|
|
7
|
+
privileged: true
|
|
8
|
+
restart: always
|
|
9
|
+
tty: true
|
|
10
|
+
stdin_open: true
|
|
11
|
+
ulimits:
|
|
12
|
+
nofile:
|
|
13
|
+
soft: ${NOFILE_LIMIT:-102400}
|
|
14
|
+
hard: ${NOFILE_LIMIT:-102400}
|
|
15
|
+
ports:
|
|
16
|
+
- "${SECONDARY_SSH_PORT}:22"
|
|
17
|
+
volumes:
|
|
18
|
+
- ./volumes/data-secondary/conf:/home/wolverine/conf
|
|
19
|
+
- ./volumes/data-secondary/logs:/home/wolverine/tmp
|
|
20
|
+
- ./volumes/ssh-shared:/home/wolverine/.ssh
|
|
21
|
+
- ./volumes/data-secondary/start.sh:/home/wolverine/start.sh
|
|
22
|
+
- ./volumes/data-secondary/healthcheck.sh:/home/wolverine/healthcheck.sh
|
|
23
|
+
- ./requirements.txt:/home/wolverine/requirements.txt:ro
|
|
24
|
+
healthcheck:
|
|
25
|
+
test: ["CMD", "/home/wolverine/healthcheck.sh"]
|
|
26
|
+
interval: 30s
|
|
27
|
+
timeout: 10s
|
|
28
|
+
retries: 5
|
|
29
|
+
start_period: 60s
|
|
30
|
+
entrypoint: []
|
|
31
|
+
command:
|
|
32
|
+
[
|
|
33
|
+
"/bin/sh",
|
|
34
|
+
"-c",
|
|
35
|
+
"chmod +x /home/wolverine/start.sh /home/wolverine/healthcheck.sh || true; if [ -s /home/wolverine/requirements.txt ]; then /home/wolverine/.pyenv/python3/bin/pip install -r /home/wolverine/requirements.txt; fi; service ssh restart; /home/wolverine/start.sh; tail -f /dev/null",
|
|
36
|
+
]
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
services:
|
|
2
|
+
wos-init:
|
|
3
|
+
image: ${PRIMARY_IMAGE}
|
|
4
|
+
platform: ${PRIMARY_PLATFORM}
|
|
5
|
+
container_name: ${INIT_HOSTNAME}
|
|
6
|
+
hostname: ${INIT_HOSTNAME}
|
|
7
|
+
privileged: true
|
|
8
|
+
tty: true
|
|
9
|
+
stdin_open: true
|
|
10
|
+
ulimits:
|
|
11
|
+
nofile:
|
|
12
|
+
soft: ${NOFILE_LIMIT:-102400}
|
|
13
|
+
hard: ${NOFILE_LIMIT:-102400}
|
|
14
|
+
environment:
|
|
15
|
+
- ENV=${ENV}
|
|
16
|
+
- NAMESPACE=${NAMESPACE}
|
|
17
|
+
- ENABLE_SECONDARY=${ENABLE_SECONDARY}
|
|
18
|
+
# The implementation selectors (wos-deploy.py INIT_SELECTOR_KEYS). Every one
|
|
19
|
+
# of them is read exactly once per instance lifetime, inside THIS container,
|
|
20
|
+
# on the boot that GENERATES start.sh; the primary re-runs that script and
|
|
21
|
+
# never re-decides. This block is therefore their authoritative landing site
|
|
22
|
+
# — the same block `re-setup` parses, so a regeneration decides the same way
|
|
23
|
+
# the original setup-* did. Deliver a value by setting it in the deploy
|
|
24
|
+
# process environment (e.g. `WOS_RING_FAMILY=rust ./wos setup-private ...`);
|
|
25
|
+
# the defaults below keep an unset selector on the image default.
|
|
26
|
+
- WOS_MIGRATE_IMPL=${WOS_MIGRATE_IMPL:-}
|
|
27
|
+
- WOS_RING_FAMILY=${WOS_RING_FAMILY:-cpp}
|
|
28
|
+
- WOS_MASTER_IMPL=${WOS_MASTER_IMPL:-}
|
|
29
|
+
- WOS_SOLDIER_IMPL=${WOS_SOLDIER_IMPL:-}
|
|
30
|
+
- WOS_BACKEND_IMPL=${WOS_BACKEND_IMPL:-}
|
|
31
|
+
- WOS_BOHR_IMPL=${WOS_BOHR_IMPL:-}
|
|
32
|
+
volumes:
|
|
33
|
+
- ./:/workspace
|
|
34
|
+
- ${DEPLOYMENT_SOURCE_DIR}:/workspace-source:ro
|
|
35
|
+
- ./volumes/data/mysql:/home/wolverine/mysql
|
|
36
|
+
- ./volumes/ssh-shared:/home/wolverine/.ssh
|
|
37
|
+
entrypoint: []
|
|
38
|
+
command: ["/bin/sh", "-c", "tail -f /dev/null"]
|
|
39
|
+
profiles:
|
|
40
|
+
- init
|
|
41
|
+
|
|
42
|
+
wos-primary:
|
|
43
|
+
image: ${PRIMARY_IMAGE}
|
|
44
|
+
platform: ${PRIMARY_PLATFORM}
|
|
45
|
+
container_name: ${PRIMARY_HOSTNAME}
|
|
46
|
+
hostname: ${PRIMARY_HOSTNAME}
|
|
47
|
+
privileged: true
|
|
48
|
+
restart: always
|
|
49
|
+
tty: true
|
|
50
|
+
stdin_open: true
|
|
51
|
+
ulimits:
|
|
52
|
+
nofile:
|
|
53
|
+
soft: ${NOFILE_LIMIT:-102400}
|
|
54
|
+
hard: ${NOFILE_LIMIT:-102400}
|
|
55
|
+
environment:
|
|
56
|
+
# storage-ring family toggle. The container
|
|
57
|
+
# supervisor (setup.py generate_start_script) reads these to pick the
|
|
58
|
+
# C++ ring (caitlyn_master/caitlyn_soldier) or the Rust ring
|
|
59
|
+
# (wos-master/wos-soldier) per role. Defaults keep the C++ ring;
|
|
60
|
+
# WOS_MASTER_IMPL / WOS_SOLDIER_IMPL override a single role when non-empty
|
|
61
|
+
# (impl = ${WOS_<ROLE>_IMPL:-${WOS_RING_FAMILY:-cpp}}).
|
|
62
|
+
- WOS_RING_FAMILY=${WOS_RING_FAMILY:-cpp}
|
|
63
|
+
- WOS_MASTER_IMPL=${WOS_MASTER_IMPL:-}
|
|
64
|
+
- WOS_SOLDIER_IMPL=${WOS_SOLDIER_IMPL:-}
|
|
65
|
+
# historical-fetch admission-control tunables for the CTM serve
|
|
66
|
+
# hub. caitlyn_time_machine resolves N (in-flight cap) / Q (backlog
|
|
67
|
+
# depth) config->env->default (8/512). Forward them here (mirroring
|
|
68
|
+
# WOS_RING_FAMILY) so a deployment tunes the cap without a rebuild; the
|
|
69
|
+
# value is carried via .env (build_env in wos-deploy.py).
|
|
70
|
+
- WOS_CTM_HISTORY_CONCURRENCY=${WOS_CTM_HISTORY_CONCURRENCY:-8}
|
|
71
|
+
- WOS_CTM_HISTORY_QUEUE_DEPTH=${WOS_CTM_HISTORY_QUEUE_DEPTH:-512}
|
|
72
|
+
# The remaining two admission-gate knobs the serve hub resolves from the
|
|
73
|
+
# environment. WEDGE_ABORT_SEC: how long the cap-level wedge sentinel
|
|
74
|
+
# tolerates a pinned-at-cap gate with static admissions before it aborts
|
|
75
|
+
# the hub (fail-fast); compiled default 1800. STATUS_SECS: the cadence of
|
|
76
|
+
# the periodic throttle-status log line (0 disables); compiled default 60.
|
|
77
|
+
# An instance that legitimately saturates the gate at boot (a large forest
|
|
78
|
+
# backfilling) needs a WEDGE_ABORT_SEC above its worst legitimate
|
|
79
|
+
# saturation, or the sentinel kills a healthy boot.
|
|
80
|
+
- WOS_CTM_HISTORY_WEDGE_ABORT_SEC=${WOS_CTM_HISTORY_WEDGE_ABORT_SEC:-1800}
|
|
81
|
+
- WOS_CTM_HISTORY_STATUS_SECS=${WOS_CTM_HISTORY_STATUS_SECS:-60}
|
|
82
|
+
# CTM boot deploy-watchdog tunables (seconds). The account plane
|
|
83
|
+
# (einstein/maxwell) starts only when every forest calculator reaches a
|
|
84
|
+
# terminal state; if the boot STALLS (a calculator never connects or hangs)
|
|
85
|
+
# the watchdog names the straggler(s) and CRASHES the CTM (fail-fast) after
|
|
86
|
+
# STALL seconds of no progress, polling every PROGRESS seconds. Carried via
|
|
87
|
+
# .env (build_env in wos-deploy.py) so a deployment tunes it without a rebuild.
|
|
88
|
+
- WOS_TM_BOOT_DEPLOY_PROGRESS_SEC=${WOS_TM_BOOT_DEPLOY_PROGRESS_SEC:-60}
|
|
89
|
+
- WOS_TM_BOOT_DEPLOY_STALL_SEC=${WOS_TM_BOOT_DEPLOY_STALL_SEC:-600}
|
|
90
|
+
# The master persist cache: its spill dir (env is the FALLBACK; the layer's
|
|
91
|
+
# generated .ini spill_dir wins), the RAM window of frames it holds before
|
|
92
|
+
# spilling, and the DISK ceiling that spill dir stops growing at. This cache is
|
|
93
|
+
# the ring's single durability layer -- dirac forwards and retains nothing, so
|
|
94
|
+
# it has neither a spill dir nor a retention switch. wos-deploy.py sets
|
|
95
|
+
# WOS_MASTER_SPILL_DIR per ring impl; the dir is bind-mounted under
|
|
96
|
+
# ./volumes/data/spill so a spilled frame survives a recreate/upgrade. Size the
|
|
97
|
+
# window from peak arrival rate x worst-case save-confirm latency: the compiled
|
|
98
|
+
# floor of 20000 frames is not a production value.
|
|
99
|
+
- WOS_MASTER_SPILL_DIR=${WOS_MASTER_SPILL_DIR:-/home/wolverine/spill/master}
|
|
100
|
+
- WOS_MASTER_RETENTION_WINDOW=${WOS_MASTER_RETENTION_WINDOW:-20000}
|
|
101
|
+
- WOS_MASTER_SPILL_HIGHWATER_BYTES=${WOS_MASTER_SPILL_HIGHWATER_BYTES:-2147483648}
|
|
102
|
+
ports:
|
|
103
|
+
- "${SSH_PORT}:22"
|
|
104
|
+
- "${DB_PORT}:3306"
|
|
105
|
+
- "${WOS_PORT_S}-${WOS_PORT_E}:3100-3105"
|
|
106
|
+
- "${HTTP_PORT}:8000"
|
|
107
|
+
- "${HTTPS_PORT}:4430"
|
|
108
|
+
# Vue (wos-web) second front-end, served by nginx on :9000.
|
|
109
|
+
- "${VUE_HTTP_PORT:-9000}:9000"
|
|
110
|
+
volumes:
|
|
111
|
+
- ./volumes/data/conf:/home/wolverine/conf
|
|
112
|
+
- ./volumes/data/conf/backend/config:/home/wolverine/backend/config
|
|
113
|
+
- ./volumes/data/mysql:/home/wolverine/mysql
|
|
114
|
+
- ./volumes/data/.my.cnf:/home/wolverine/.my.cnf
|
|
115
|
+
- ./volumes/data/backtests:/home/wolverine/backtests
|
|
116
|
+
- ./volumes/data/logs:/home/wolverine/tmp
|
|
117
|
+
- ./volumes/data/CA:/home/wolverine/CA
|
|
118
|
+
# the PER-LAYER durable spill tree (spill/{dirac,master,
|
|
119
|
+
# wos-master}) under the persistent volumes/ tree so spilled frames survive
|
|
120
|
+
# a container upgrade/recreate. wos-deploy.py provisions the per-layer subdirs.
|
|
121
|
+
- ./volumes/data/spill:/home/wolverine/spill
|
|
122
|
+
- ./volumes/ssh-shared:/home/wolverine/.ssh
|
|
123
|
+
- ./volumes/data/start.sh:/home/wolverine/start.sh
|
|
124
|
+
- ./volumes/data/healthcheck.sh:/home/wolverine/healthcheck.sh
|
|
125
|
+
- ./requirements.txt:/home/wolverine/requirements.txt:ro
|
|
126
|
+
healthcheck:
|
|
127
|
+
test: ["CMD", "/home/wolverine/healthcheck.sh"]
|
|
128
|
+
interval: 30s
|
|
129
|
+
timeout: 10s
|
|
130
|
+
retries: 5
|
|
131
|
+
start_period: 60s
|
|
132
|
+
entrypoint: []
|
|
133
|
+
command:
|
|
134
|
+
[
|
|
135
|
+
"/bin/sh",
|
|
136
|
+
"-c",
|
|
137
|
+
"chmod +x /home/wolverine/start.sh /home/wolverine/healthcheck.sh || true; if [ -s /home/wolverine/requirements.txt ]; then /home/wolverine/.pyenv/python3/bin/pip install -r /home/wolverine/requirements.txt; fi; /usr/bin/entrypoint.sh; tail -f /dev/null",
|
|
138
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Deployment scripts package."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Deployment function package."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Shared function implementations."""
|