vortex-cli 6.2.1__tar.gz → 6.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/PKG-INFO +14 -2
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/README.md +13 -1
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/pyproject.toml +1 -1
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/cli.py +26 -1
- vortex_cli-6.3.0/vortex/commands/push.py +303 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/main.py +8 -1
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex_cli.egg-info/PKG-INFO +14 -2
- vortex_cli-6.2.1/vortex/commands/push.py +0 -132
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/LICENSE +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/setup.cfg +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/__init__.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/__main__.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/colour.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/__init__.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/agenda.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/clean.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/clone.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/code.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/compile.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/config.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/copy.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/db.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/delete.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/docs.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/execute.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/export.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/find.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/grep.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/import_.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/libs.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/list.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/log.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/new.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/pull.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/render.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/schema.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/status.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/undo.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/use.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/commands/watch.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/constants.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/docs/Blackbook v2.md +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/docs/Blackbook.pdf +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/docs/index.html +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/docs/marked.min.js +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/gateway.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/lib/puakma-6.0.40.jar +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/libs.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/logging.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/models.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/soap.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/spinner.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/util.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/webdesign.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex/workspace.py +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex_cli.egg-info/SOURCES.txt +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex_cli.egg-info/dependency_links.txt +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex_cli.egg-info/entry_points.txt +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex_cli.egg-info/requires.txt +0 -0
- {vortex_cli-6.2.1 → vortex_cli-6.3.0}/vortex_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vortex_cli
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.0
|
|
4
4
|
Summary: Vortex CLI
|
|
5
5
|
Author-email: Jordan Amos <jordan.amos@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -108,6 +108,18 @@ While it is possible to use without it, this software has been purposefully desi
|
|
|
108
108
|
java_environment_name = JavaSE-17 ; Java Execution Environment name https://docs.osgi.org/reference/eenames.html
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
+
## Upgrading to 6.3
|
|
112
|
+
|
|
113
|
+
- **`vortex push` only uploads what changed.** On a `backend = gateway` server the push first
|
|
114
|
+
reads the server's element hashes and skips every file that already matches - a full-app push
|
|
115
|
+
that used to re-send (and journal) hundreds of identical files now sends the few that differ.
|
|
116
|
+
`--all` restores the old push-everything behaviour; `backend = soap` still pushes everything.
|
|
117
|
+
- **Uploads run concurrently** - up to `--jobs N` elements at once (default 8; `-j 1` is the
|
|
118
|
+
old one-at-a-time push). A Java element's source and class still go in that order, and a
|
|
119
|
+
failed source stops that element's class from being uploaded over it.
|
|
120
|
+
- **`push --name` prefers an exact match.** Previously `--name dashboard` also pushed a
|
|
121
|
+
`Dashboard` element; now the case-insensitive match only applies when nothing matches exactly.
|
|
122
|
+
|
|
111
123
|
## Upgrading to 6.0
|
|
112
124
|
|
|
113
125
|
6.0 is additive - **the default behaviour is unchanged**. Every server keeps using the SOAP
|
|
@@ -163,7 +175,7 @@ For a full list of commands see `--help`.
|
|
|
163
175
|
- `log`: View the server log.
|
|
164
176
|
- `status`: Show the server status - via the agent gateway when available, otherwise the raw console `status` output.
|
|
165
177
|
- `agenda`: List every scheduled action with its decoded schedule, last and next run, and whether it is overdue (read-only, gateway only).
|
|
166
|
-
- `push`: Upload local design files (source, compiled classes, pages, resources) to the server without a watch session - journaled and deploy-confirmed when routed via the gateway.
|
|
178
|
+
- `push`: Upload local design files (source, compiled classes, pages, resources) to the server without a watch session - journaled and deploy-confirmed when routed via the gateway. On a gateway server only files whose content differs from the server's are uploaded (compared by the server's element hashes; `--all` pushes everything), and elements upload concurrently (`--jobs N`, default 8) while each Java element's source always lands before its class. `--name` matches exactly first, case-insensitively only when nothing matches exactly.
|
|
167
179
|
- `pull`: Refresh cloned applications in place via the gateway's incremental sync - only elements changed since the last sync are downloaded. Refuses to overwrite locally modified files without `--force`.
|
|
168
180
|
- `render`: Render a PAGE server-side as your identity and print it (or `--out FILE`) - see a change without a browser login (gateway only).
|
|
169
181
|
- `undo`: The server-side undo journal. No arguments lists journal entries; with an entry id it dry-runs a restore, and `--confirm` writes the journaled version back (gateway only).
|
|
@@ -64,6 +64,18 @@ While it is possible to use without it, this software has been purposefully desi
|
|
|
64
64
|
java_environment_name = JavaSE-17 ; Java Execution Environment name https://docs.osgi.org/reference/eenames.html
|
|
65
65
|
```
|
|
66
66
|
|
|
67
|
+
## Upgrading to 6.3
|
|
68
|
+
|
|
69
|
+
- **`vortex push` only uploads what changed.** On a `backend = gateway` server the push first
|
|
70
|
+
reads the server's element hashes and skips every file that already matches - a full-app push
|
|
71
|
+
that used to re-send (and journal) hundreds of identical files now sends the few that differ.
|
|
72
|
+
`--all` restores the old push-everything behaviour; `backend = soap` still pushes everything.
|
|
73
|
+
- **Uploads run concurrently** - up to `--jobs N` elements at once (default 8; `-j 1` is the
|
|
74
|
+
old one-at-a-time push). A Java element's source and class still go in that order, and a
|
|
75
|
+
failed source stops that element's class from being uploaded over it.
|
|
76
|
+
- **`push --name` prefers an exact match.** Previously `--name dashboard` also pushed a
|
|
77
|
+
`Dashboard` element; now the case-insensitive match only applies when nothing matches exactly.
|
|
78
|
+
|
|
67
79
|
## Upgrading to 6.0
|
|
68
80
|
|
|
69
81
|
6.0 is additive - **the default behaviour is unchanged**. Every server keeps using the SOAP
|
|
@@ -119,7 +131,7 @@ For a full list of commands see `--help`.
|
|
|
119
131
|
- `log`: View the server log.
|
|
120
132
|
- `status`: Show the server status - via the agent gateway when available, otherwise the raw console `status` output.
|
|
121
133
|
- `agenda`: List every scheduled action with its decoded schedule, last and next run, and whether it is overdue (read-only, gateway only).
|
|
122
|
-
- `push`: Upload local design files (source, compiled classes, pages, resources) to the server without a watch session - journaled and deploy-confirmed when routed via the gateway.
|
|
134
|
+
- `push`: Upload local design files (source, compiled classes, pages, resources) to the server without a watch session - journaled and deploy-confirmed when routed via the gateway. On a gateway server only files whose content differs from the server's are uploaded (compared by the server's element hashes; `--all` pushes everything), and elements upload concurrently (`--jobs N`, default 8) while each Java element's source always lands before its class. `--name` matches exactly first, case-insensitively only when nothing matches exactly.
|
|
123
135
|
- `pull`: Refresh cloned applications in place via the gateway's incremental sync - only elements changed since the last sync are downloaded. Refuses to overwrite locally modified files without `--force`.
|
|
124
136
|
- `render`: Render a PAGE server-side as your identity and print it (or `--out FILE`) - see a change without a browser login (gateway only).
|
|
125
137
|
- `undo`: The server-side undo journal. No arguments lists journal entries; with an entry id it dry-runs a restore, and `--confirm` writes the journaled version back (gateway only).
|
|
@@ -14,6 +14,7 @@ from typing import NamedTuple
|
|
|
14
14
|
from vortex.commands.db import DEFAULT_LIMIT_N_RESULTS
|
|
15
15
|
from vortex.commands.db import MAX_LIMIT_N_RESULTS
|
|
16
16
|
from vortex.commands.db import MIN_LIMIT_N_RESULTS
|
|
17
|
+
from vortex.commands.push import DEFAULT_JOBS as DEFAULT_PUSH_JOBS
|
|
17
18
|
from vortex.models import DesignType
|
|
18
19
|
from vortex.models import PuakmaApplication
|
|
19
20
|
|
|
@@ -718,7 +719,31 @@ def add_push_parser(command_parser: _SubParsersAction[ArgumentParser]) -> None:
|
|
|
718
719
|
"-n",
|
|
719
720
|
dest="name_filter",
|
|
720
721
|
default=None,
|
|
721
|
-
help=
|
|
722
|
+
help=(
|
|
723
|
+
"Only push the design element with this name (an exact match "
|
|
724
|
+
"wins; case-insensitive only when nothing matches exactly)"
|
|
725
|
+
),
|
|
726
|
+
)
|
|
727
|
+
push_parser.add_argument(
|
|
728
|
+
"--all",
|
|
729
|
+
"-a",
|
|
730
|
+
action="store_true",
|
|
731
|
+
help=(
|
|
732
|
+
"Push every local file, including ones whose content already "
|
|
733
|
+
"matches the server. By default (gateway backend) unchanged "
|
|
734
|
+
"files are skipped using the server's element hashes"
|
|
735
|
+
),
|
|
736
|
+
)
|
|
737
|
+
push_parser.add_argument(
|
|
738
|
+
"--jobs",
|
|
739
|
+
"-j",
|
|
740
|
+
type=int,
|
|
741
|
+
default=DEFAULT_PUSH_JOBS,
|
|
742
|
+
metavar="N",
|
|
743
|
+
help=(
|
|
744
|
+
"Maximum elements uploaded concurrently "
|
|
745
|
+
f"(default {DEFAULT_PUSH_JOBS}; 1 uploads one at a time)"
|
|
746
|
+
),
|
|
722
747
|
)
|
|
723
748
|
_add_server_option(push_parser)
|
|
724
749
|
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"""
|
|
2
|
+
vortex push - upload local design files without a watch session.
|
|
3
|
+
|
|
4
|
+
Two things keep a push fast:
|
|
5
|
+
|
|
6
|
+
1. Only what changed is uploaded. On backend=gateway the server's
|
|
7
|
+
element inventory carries a SHA-1 per blob ('elements?hashes=1', the
|
|
8
|
+
same primitive 'vortex pull' diffs with), so an upload whose local
|
|
9
|
+
bytes already hash to what the server holds is skipped. A full-app
|
|
10
|
+
push that used to re-send every file (and journal every one of them)
|
|
11
|
+
now sends the handful that differ. '--all' restores the old
|
|
12
|
+
everything-goes behaviour; backend=soap has no hashes and always
|
|
13
|
+
pushes everything.
|
|
14
|
+
2. Uploads run concurrently, bounded by '--jobs'. The per-upload cost is
|
|
15
|
+
almost entirely the gateway's own journal + write + rehash, not the
|
|
16
|
+
round trip, and the gateway handles parallel requests fine ('vortex
|
|
17
|
+
watch' has always uploaded a save batch concurrently). The one order
|
|
18
|
+
that matters is kept: the blobs of ONE element (a Java source and its
|
|
19
|
+
compiled class) go strictly source-then-class - the journal keys
|
|
20
|
+
entries by millisecond + element, and a class must never land
|
|
21
|
+
without its source.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import asyncio
|
|
27
|
+
import base64
|
|
28
|
+
import hashlib
|
|
29
|
+
import logging
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from vortex.commands.watch import aupload_design_routed
|
|
33
|
+
from vortex.gateway import GatewayError
|
|
34
|
+
from vortex.gateway import GatewayUnavailable
|
|
35
|
+
from vortex.models import DesignObject
|
|
36
|
+
from vortex.models import DesignType
|
|
37
|
+
from vortex.models import PuakmaApplication
|
|
38
|
+
from vortex.models import PuakmaServer
|
|
39
|
+
from vortex.util import confirm_protected
|
|
40
|
+
from vortex.workspace import Workspace
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger("vortex")
|
|
43
|
+
|
|
44
|
+
# Concurrent uploads in flight. Measured against a real gateway: 20
|
|
45
|
+
# concurrent requests complete in the time of two sequential ones, so
|
|
46
|
+
# the ceiling is politeness to the server, not throughput
|
|
47
|
+
DEFAULT_JOBS = 8
|
|
48
|
+
# 'elements?hashes=1' reads every blob server-side - allow for big apps
|
|
49
|
+
HASHES_TIMEOUT_SECONDS = 300
|
|
50
|
+
|
|
51
|
+
# (element, upload_source): SOURCE when True, DATA when False
|
|
52
|
+
Upload = tuple[DesignObject, bool]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _b64_sha1(b64_blob: str) -> str:
|
|
56
|
+
"""The gateway's hash encoding (base64 of SHA-1) of a base64 blob"""
|
|
57
|
+
return base64.b64encode(hashlib.sha1(base64.b64decode(b64_blob)).digest()).decode()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _select(objs: list[DesignObject], name_filter: str | None) -> list[DesignObject]:
|
|
61
|
+
"""
|
|
62
|
+
The elements '--name' refers to. An exact name wins outright: an
|
|
63
|
+
app can hold both 'Dashboard' (SHARED_CODE) and 'dashboard' (PAGE),
|
|
64
|
+
and asking for one must not push the other. Only when nothing
|
|
65
|
+
matches exactly is the name matched case-insensitively.
|
|
66
|
+
"""
|
|
67
|
+
if not name_filter:
|
|
68
|
+
return objs
|
|
69
|
+
exact = [o for o in objs if o.name == name_filter]
|
|
70
|
+
if exact:
|
|
71
|
+
return exact
|
|
72
|
+
wanted = name_filter.casefold()
|
|
73
|
+
return [o for o in objs if o.name.casefold() == wanted]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _collect(
|
|
77
|
+
workspace: Workspace, app: PuakmaApplication, name_filter: str | None
|
|
78
|
+
) -> tuple[list[Upload], bool]:
|
|
79
|
+
"""
|
|
80
|
+
(object, upload_source) pairs for every design object with a local
|
|
81
|
+
file: SOURCE from .java/DOCUMENTATION files, DATA from compiled
|
|
82
|
+
zbin classes and from PAGE/RESOURCE files. An element can appear
|
|
83
|
+
twice (java source + compiled class), source first. An element whose
|
|
84
|
+
class references nested classes is refused ENTIRELY - pushing its
|
|
85
|
+
source without the class would leave the server running old
|
|
86
|
+
bytecode with new source attached (silent drift) - and the second
|
|
87
|
+
return value reports that a refusal happened so push can exit
|
|
88
|
+
non-zero.
|
|
89
|
+
"""
|
|
90
|
+
uploads: list[Upload] = []
|
|
91
|
+
refused = False
|
|
92
|
+
for obj in _select(app.design_objects, name_filter):
|
|
93
|
+
if obj.is_jar_library:
|
|
94
|
+
continue
|
|
95
|
+
design_path = obj.design_path(workspace).path
|
|
96
|
+
upload_source = (
|
|
97
|
+
obj.design_type.is_java_type or obj.design_type == DesignType.DOCUMENTATION
|
|
98
|
+
)
|
|
99
|
+
if design_path.exists():
|
|
100
|
+
file_bytes = design_path.read_bytes()
|
|
101
|
+
if upload_source:
|
|
102
|
+
obj.design_source = file_bytes
|
|
103
|
+
else:
|
|
104
|
+
obj.design_data = file_bytes
|
|
105
|
+
uploads.append((obj, upload_source))
|
|
106
|
+
if obj.design_type.is_java_type:
|
|
107
|
+
class_file = workspace.path / app.path / "zbin" / f"{obj.name}.class"
|
|
108
|
+
if class_file.exists():
|
|
109
|
+
nested = sorted(
|
|
110
|
+
c.name for c in class_file.parent.glob(f"{class_file.stem}$*.class")
|
|
111
|
+
)
|
|
112
|
+
if nested:
|
|
113
|
+
logger.error(
|
|
114
|
+
f"Not pushing '{obj.name}' at all: its class "
|
|
115
|
+
f"references nested classes ({', '.join(nested)}) "
|
|
116
|
+
"which are never uploaded and would fail on the "
|
|
117
|
+
"server. Refactor them to top-level SHARED_CODE "
|
|
118
|
+
"classes first"
|
|
119
|
+
)
|
|
120
|
+
uploads = [u for u in uploads if u[0] is not obj]
|
|
121
|
+
refused = True
|
|
122
|
+
continue
|
|
123
|
+
obj.design_data = class_file.read_bytes()
|
|
124
|
+
uploads.append((obj, False))
|
|
125
|
+
return uploads, refused
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _drop_unchanged(
|
|
129
|
+
client: Any,
|
|
130
|
+
app: PuakmaApplication,
|
|
131
|
+
uploads: list[Upload],
|
|
132
|
+
timeout: int = HASHES_TIMEOUT_SECONDS,
|
|
133
|
+
) -> tuple[list[Upload], int]:
|
|
134
|
+
"""
|
|
135
|
+
The uploads whose bytes differ from what the server holds, and how
|
|
136
|
+
many were dropped as identical. One 'elements?hashes=1' request per
|
|
137
|
+
app. An element the server no longer has, or a blob it reports no
|
|
138
|
+
hash for (SQL NULL), is never dropped - the upload itself gives the
|
|
139
|
+
real answer. If the inventory can't be read the whole list is kept:
|
|
140
|
+
a push must never silently do less than it was asked to.
|
|
141
|
+
"""
|
|
142
|
+
try:
|
|
143
|
+
reply = client.get_elements(app.id, hashes=True, timeout=timeout)
|
|
144
|
+
by_id = {int(e["id"]): e for e in reply["elements"]}
|
|
145
|
+
except (GatewayError, KeyError, TypeError, ValueError) as e:
|
|
146
|
+
logger.warning(
|
|
147
|
+
f"[{app}] Couldn't read the server's element hashes ({e}) - "
|
|
148
|
+
"pushing every local file"
|
|
149
|
+
)
|
|
150
|
+
return uploads, 0
|
|
151
|
+
kept: list[Upload] = []
|
|
152
|
+
dropped = 0
|
|
153
|
+
for obj, upload_source in uploads:
|
|
154
|
+
element = by_id.get(obj.id)
|
|
155
|
+
server_hash = (
|
|
156
|
+
element.get("sourceHash" if upload_source else "dataHash")
|
|
157
|
+
if element is not None
|
|
158
|
+
else None
|
|
159
|
+
)
|
|
160
|
+
local_b64 = obj._design_source if upload_source else obj._design_data
|
|
161
|
+
if server_hash and str(server_hash) == _b64_sha1(local_b64):
|
|
162
|
+
logger.debug(
|
|
163
|
+
f"[{app}] {'SOURCE' if upload_source else 'DATA'} of {obj} "
|
|
164
|
+
"already matches the server - skipped"
|
|
165
|
+
)
|
|
166
|
+
dropped += 1
|
|
167
|
+
else:
|
|
168
|
+
kept.append((obj, upload_source))
|
|
169
|
+
return kept, dropped
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _chains(uploads: list[Upload]) -> list[list[Upload]]:
|
|
173
|
+
"""
|
|
174
|
+
Uploads grouped per element, in their collected order (source before
|
|
175
|
+
class). Chains run concurrently with each other; within a chain the
|
|
176
|
+
uploads are strictly sequential.
|
|
177
|
+
"""
|
|
178
|
+
by_element: dict[tuple[int, int], list[Upload]] = {}
|
|
179
|
+
for obj, upload_source in uploads:
|
|
180
|
+
by_element.setdefault((obj.app.id, obj.id), []).append((obj, upload_source))
|
|
181
|
+
return list(by_element.values())
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
async def _apush(
|
|
185
|
+
server: PuakmaServer, uploads: list[Upload], jobs: int = DEFAULT_JOBS
|
|
186
|
+
) -> int:
|
|
187
|
+
"""
|
|
188
|
+
Upload everything, at most `jobs` elements in flight at once. A
|
|
189
|
+
failed blob stops its element's chain: a class is never uploaded
|
|
190
|
+
over a source that didn't make it (and vice versa), so an element
|
|
191
|
+
is either fully updated or left as it was - but one element's
|
|
192
|
+
failure never stops the others.
|
|
193
|
+
|
|
194
|
+
That last guarantee has to hold for a RAISED failure too, not just
|
|
195
|
+
a False one. aupload_design_routed turns a gateway refusal/outage
|
|
196
|
+
into False itself, but the webdesign and SOAP paths can still raise
|
|
197
|
+
(a SOAP fault, an unexpected transport error, the handshake being
|
|
198
|
+
rejected). A bare gather would abandon every other chain mid-flight
|
|
199
|
+
and then close the HTTP client out from under those in-flight
|
|
200
|
+
uploads on the way out of 'async with server', so exceptions are
|
|
201
|
+
collected and reported per element instead.
|
|
202
|
+
"""
|
|
203
|
+
semaphore = asyncio.Semaphore(max(1, jobs))
|
|
204
|
+
chains = _chains(uploads)
|
|
205
|
+
|
|
206
|
+
async def run_chain(chain: list[Upload]) -> bool:
|
|
207
|
+
async with semaphore:
|
|
208
|
+
for obj, upload_source in chain:
|
|
209
|
+
if not await aupload_design_routed(server, obj, upload_source):
|
|
210
|
+
return False
|
|
211
|
+
return True
|
|
212
|
+
|
|
213
|
+
async with server:
|
|
214
|
+
# No eager SOAP handshake: each upload routes itself and only
|
|
215
|
+
# the ones taking the SOAP path connect (aupload_design_routed)
|
|
216
|
+
results = await asyncio.gather(
|
|
217
|
+
*(run_chain(c) for c in chains), return_exceptions=True
|
|
218
|
+
)
|
|
219
|
+
ret = 0
|
|
220
|
+
for chain, result in zip(chains, results, strict=True):
|
|
221
|
+
if isinstance(result, BaseException):
|
|
222
|
+
if not isinstance(result, Exception):
|
|
223
|
+
# Ctrl-C / cancellation is not an upload failure
|
|
224
|
+
raise result
|
|
225
|
+
logger.error(
|
|
226
|
+
f"[{server.name}] Upload of {chain[0][0]} failed: "
|
|
227
|
+
f"{type(result).__name__}: {result}"
|
|
228
|
+
)
|
|
229
|
+
ret = 1
|
|
230
|
+
elif not result:
|
|
231
|
+
ret = 1
|
|
232
|
+
return ret
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def push(
|
|
236
|
+
workspace: Workspace,
|
|
237
|
+
server: PuakmaServer,
|
|
238
|
+
app_ids: list[int],
|
|
239
|
+
name_filter: str | None,
|
|
240
|
+
*,
|
|
241
|
+
push_all: bool = False,
|
|
242
|
+
jobs: int = DEFAULT_JOBS,
|
|
243
|
+
timeout: int = HASHES_TIMEOUT_SECONDS,
|
|
244
|
+
) -> int:
|
|
245
|
+
"""
|
|
246
|
+
Push local design files to the server without a watch session (idea
|
|
247
|
+
V2 - closes the 'created outside watch = empty server-side source'
|
|
248
|
+
gap). Uploads go via the agent gateway when negotiated (journaled,
|
|
249
|
+
deploy-confirmed, unchanged files skipped), otherwise SOAP.
|
|
250
|
+
"""
|
|
251
|
+
# '-j 0' / '-j -1' mean one at a time, not zero uploads - clamp here
|
|
252
|
+
# so the count that is logged is the count that is used
|
|
253
|
+
jobs = max(1, jobs)
|
|
254
|
+
apps = [app for app in workspace.listapps(server) if app.id in set(app_ids)]
|
|
255
|
+
missing = set(app_ids) - {app.id for app in apps}
|
|
256
|
+
if missing:
|
|
257
|
+
logger.error(f"No locally cloned app(s) with id(s): {sorted(missing)}")
|
|
258
|
+
return 1
|
|
259
|
+
|
|
260
|
+
try:
|
|
261
|
+
client = server.negotiate_gateway()
|
|
262
|
+
except GatewayUnavailable as e:
|
|
263
|
+
# backend=gateway and no usable gateway: nothing is pushed
|
|
264
|
+
# (no SOAP fallback - the journal is the point of the route)
|
|
265
|
+
logger.error(f"[{server.name}] {e} - nothing pushed")
|
|
266
|
+
return 1
|
|
267
|
+
|
|
268
|
+
uploads: list[Upload] = []
|
|
269
|
+
any_refused = False
|
|
270
|
+
skipped = 0
|
|
271
|
+
for app in apps:
|
|
272
|
+
app_uploads, refused = _collect(workspace, app, name_filter)
|
|
273
|
+
any_refused = any_refused or refused
|
|
274
|
+
if app_uploads and client is not None and not push_all:
|
|
275
|
+
app_uploads, dropped = _drop_unchanged(client, app, app_uploads, timeout)
|
|
276
|
+
skipped += dropped
|
|
277
|
+
uploads.extend(app_uploads)
|
|
278
|
+
if not uploads:
|
|
279
|
+
if skipped:
|
|
280
|
+
logger.info(
|
|
281
|
+
f"Nothing to push: {skipped} upload(s) already match the "
|
|
282
|
+
"server (--all pushes them anyway)"
|
|
283
|
+
)
|
|
284
|
+
return 1 if any_refused else 0
|
|
285
|
+
logger.error("Nothing to push (no matching local design files)")
|
|
286
|
+
return 1
|
|
287
|
+
|
|
288
|
+
if not confirm_protected(server, f"push {len(uploads)} design upload(s)"):
|
|
289
|
+
return 1
|
|
290
|
+
unchanged = f", {skipped} unchanged skipped" if skipped else ""
|
|
291
|
+
logger.info(
|
|
292
|
+
f"Pushing {len(uploads)} design upload(s){unchanged} ({jobs} at a time)..."
|
|
293
|
+
)
|
|
294
|
+
with workspace.app_locks((app.host, app.id) for app in apps):
|
|
295
|
+
ret = asyncio.run(_apush(server, uploads, jobs))
|
|
296
|
+
if ret == 0:
|
|
297
|
+
# re-pickle the manifests so they keep tracking what the
|
|
298
|
+
# server holds - the baseline 'vortex pull' diffs against
|
|
299
|
+
for app in apps:
|
|
300
|
+
workspace.mkdir(app)
|
|
301
|
+
# A refused element (nested classes) is a failure even when every
|
|
302
|
+
# other upload succeeded - scripts and agents must see it
|
|
303
|
+
return 1 if any_refused else ret
|
|
@@ -359,7 +359,14 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
359
359
|
elif args.command == "agenda":
|
|
360
360
|
return agenda(server, args.show_all)
|
|
361
361
|
elif args.command == "push":
|
|
362
|
-
return push(
|
|
362
|
+
return push(
|
|
363
|
+
workspace,
|
|
364
|
+
server,
|
|
365
|
+
args.app_ids,
|
|
366
|
+
args.name_filter,
|
|
367
|
+
push_all=args.all,
|
|
368
|
+
jobs=args.jobs,
|
|
369
|
+
)
|
|
363
370
|
elif args.command == "pull":
|
|
364
371
|
return pull(
|
|
365
372
|
workspace,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vortex_cli
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.0
|
|
4
4
|
Summary: Vortex CLI
|
|
5
5
|
Author-email: Jordan Amos <jordan.amos@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -108,6 +108,18 @@ While it is possible to use without it, this software has been purposefully desi
|
|
|
108
108
|
java_environment_name = JavaSE-17 ; Java Execution Environment name https://docs.osgi.org/reference/eenames.html
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
+
## Upgrading to 6.3
|
|
112
|
+
|
|
113
|
+
- **`vortex push` only uploads what changed.** On a `backend = gateway` server the push first
|
|
114
|
+
reads the server's element hashes and skips every file that already matches - a full-app push
|
|
115
|
+
that used to re-send (and journal) hundreds of identical files now sends the few that differ.
|
|
116
|
+
`--all` restores the old push-everything behaviour; `backend = soap` still pushes everything.
|
|
117
|
+
- **Uploads run concurrently** - up to `--jobs N` elements at once (default 8; `-j 1` is the
|
|
118
|
+
old one-at-a-time push). A Java element's source and class still go in that order, and a
|
|
119
|
+
failed source stops that element's class from being uploaded over it.
|
|
120
|
+
- **`push --name` prefers an exact match.** Previously `--name dashboard` also pushed a
|
|
121
|
+
`Dashboard` element; now the case-insensitive match only applies when nothing matches exactly.
|
|
122
|
+
|
|
111
123
|
## Upgrading to 6.0
|
|
112
124
|
|
|
113
125
|
6.0 is additive - **the default behaviour is unchanged**. Every server keeps using the SOAP
|
|
@@ -163,7 +175,7 @@ For a full list of commands see `--help`.
|
|
|
163
175
|
- `log`: View the server log.
|
|
164
176
|
- `status`: Show the server status - via the agent gateway when available, otherwise the raw console `status` output.
|
|
165
177
|
- `agenda`: List every scheduled action with its decoded schedule, last and next run, and whether it is overdue (read-only, gateway only).
|
|
166
|
-
- `push`: Upload local design files (source, compiled classes, pages, resources) to the server without a watch session - journaled and deploy-confirmed when routed via the gateway.
|
|
178
|
+
- `push`: Upload local design files (source, compiled classes, pages, resources) to the server without a watch session - journaled and deploy-confirmed when routed via the gateway. On a gateway server only files whose content differs from the server's are uploaded (compared by the server's element hashes; `--all` pushes everything), and elements upload concurrently (`--jobs N`, default 8) while each Java element's source always lands before its class. `--name` matches exactly first, case-insensitively only when nothing matches exactly.
|
|
167
179
|
- `pull`: Refresh cloned applications in place via the gateway's incremental sync - only elements changed since the last sync are downloaded. Refuses to overwrite locally modified files without `--force`.
|
|
168
180
|
- `render`: Render a PAGE server-side as your identity and print it (or `--out FILE`) - see a change without a browser login (gateway only).
|
|
169
181
|
- `undo`: The server-side undo journal. No arguments lists journal entries; with an entry id it dry-runs a restore, and `--confirm` writes the journaled version back (gateway only).
|
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import asyncio
|
|
4
|
-
import logging
|
|
5
|
-
|
|
6
|
-
from vortex.commands.watch import aupload_design_routed
|
|
7
|
-
from vortex.models import DesignObject
|
|
8
|
-
from vortex.models import DesignType
|
|
9
|
-
from vortex.models import PuakmaApplication
|
|
10
|
-
from vortex.models import PuakmaServer
|
|
11
|
-
from vortex.util import confirm_protected
|
|
12
|
-
from vortex.workspace import Workspace
|
|
13
|
-
|
|
14
|
-
logger = logging.getLogger("vortex")
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def _collect(
|
|
18
|
-
workspace: Workspace, app: PuakmaApplication, name_filter: str | None
|
|
19
|
-
) -> tuple[list[tuple[DesignObject, bool]], bool]:
|
|
20
|
-
"""
|
|
21
|
-
(object, upload_source) pairs for every design object with a local
|
|
22
|
-
file: SOURCE from .java/DOCUMENTATION files, DATA from compiled
|
|
23
|
-
zbin classes and from PAGE/RESOURCE files. An element can appear
|
|
24
|
-
twice (java source + compiled class). An element whose class
|
|
25
|
-
references nested classes is refused ENTIRELY - pushing its source
|
|
26
|
-
without the class would leave the server running old bytecode with
|
|
27
|
-
new source attached (silent drift) - and the second return value
|
|
28
|
-
reports that a refusal happened so push can exit non-zero.
|
|
29
|
-
"""
|
|
30
|
-
uploads: list[tuple[DesignObject, bool]] = []
|
|
31
|
-
refused = False
|
|
32
|
-
for obj in app.design_objects:
|
|
33
|
-
if name_filter and obj.name.casefold() != name_filter.casefold():
|
|
34
|
-
continue
|
|
35
|
-
if obj.is_jar_library:
|
|
36
|
-
continue
|
|
37
|
-
design_path = obj.design_path(workspace).path
|
|
38
|
-
upload_source = (
|
|
39
|
-
obj.design_type.is_java_type or obj.design_type == DesignType.DOCUMENTATION
|
|
40
|
-
)
|
|
41
|
-
if design_path.exists():
|
|
42
|
-
file_bytes = design_path.read_bytes()
|
|
43
|
-
if upload_source:
|
|
44
|
-
obj.design_source = file_bytes
|
|
45
|
-
else:
|
|
46
|
-
obj.design_data = file_bytes
|
|
47
|
-
uploads.append((obj, upload_source))
|
|
48
|
-
if obj.design_type.is_java_type:
|
|
49
|
-
class_file = workspace.path / app.path / "zbin" / f"{obj.name}.class"
|
|
50
|
-
if class_file.exists():
|
|
51
|
-
nested = sorted(
|
|
52
|
-
c.name for c in class_file.parent.glob(f"{class_file.stem}$*.class")
|
|
53
|
-
)
|
|
54
|
-
if nested:
|
|
55
|
-
logger.error(
|
|
56
|
-
f"Not pushing '{obj.name}' at all: its class "
|
|
57
|
-
f"references nested classes ({', '.join(nested)}) "
|
|
58
|
-
"which are never uploaded and would fail on the "
|
|
59
|
-
"server. Refactor them to top-level SHARED_CODE "
|
|
60
|
-
"classes first"
|
|
61
|
-
)
|
|
62
|
-
uploads = [u for u in uploads if u[0] is not obj]
|
|
63
|
-
refused = True
|
|
64
|
-
continue
|
|
65
|
-
obj.design_data = class_file.read_bytes()
|
|
66
|
-
uploads.append((obj, False))
|
|
67
|
-
return uploads, refused
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
async def _apush(server: PuakmaServer, uploads: list[tuple[DesignObject, bool]]) -> int:
|
|
71
|
-
from vortex.gateway import GatewayUnavailable
|
|
72
|
-
|
|
73
|
-
ret = 0
|
|
74
|
-
async with server as s:
|
|
75
|
-
# No eager SOAP handshake: each upload routes itself and only
|
|
76
|
-
# the ones taking the SOAP path connect (aupload_design_routed).
|
|
77
|
-
# On a gateway server the acting identity may have no SOAP
|
|
78
|
-
# access at all
|
|
79
|
-
try:
|
|
80
|
-
await asyncio.to_thread(s.negotiate_gateway)
|
|
81
|
-
except GatewayUnavailable as e:
|
|
82
|
-
# backend=gateway and no usable gateway: nothing is pushed
|
|
83
|
-
# (no SOAP fallback - the journal is the point of the route)
|
|
84
|
-
logger.error(f"[{s.name}] {e} - nothing pushed")
|
|
85
|
-
return 1
|
|
86
|
-
for obj, upload_source in uploads:
|
|
87
|
-
ok = await aupload_design_routed(s, obj, upload_source)
|
|
88
|
-
ret |= 0 if ok else 1
|
|
89
|
-
return ret
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
def push(
|
|
93
|
-
workspace: Workspace,
|
|
94
|
-
server: PuakmaServer,
|
|
95
|
-
app_ids: list[int],
|
|
96
|
-
name_filter: str | None,
|
|
97
|
-
) -> int:
|
|
98
|
-
"""
|
|
99
|
-
Push local design files to the server without a watch session (idea
|
|
100
|
-
V2 - closes the 'created outside watch = empty server-side source'
|
|
101
|
-
gap). Uploads go via the agent gateway when negotiated (journaled,
|
|
102
|
-
deploy-confirmed), otherwise SOAP.
|
|
103
|
-
"""
|
|
104
|
-
apps = [app for app in workspace.listapps(server) if app.id in set(app_ids)]
|
|
105
|
-
missing = set(app_ids) - {app.id for app in apps}
|
|
106
|
-
if missing:
|
|
107
|
-
logger.error(f"No locally cloned app(s) with id(s): {sorted(missing)}")
|
|
108
|
-
return 1
|
|
109
|
-
|
|
110
|
-
uploads: list[tuple[DesignObject, bool]] = []
|
|
111
|
-
any_refused = False
|
|
112
|
-
for app in apps:
|
|
113
|
-
app_uploads, refused = _collect(workspace, app, name_filter)
|
|
114
|
-
uploads.extend(app_uploads)
|
|
115
|
-
any_refused = any_refused or refused
|
|
116
|
-
if not uploads:
|
|
117
|
-
logger.error("Nothing to push (no matching local design files)")
|
|
118
|
-
return 1
|
|
119
|
-
|
|
120
|
-
if not confirm_protected(server, f"push {len(uploads)} design upload(s)"):
|
|
121
|
-
return 1
|
|
122
|
-
logger.info(f"Pushing {len(uploads)} design upload(s)...")
|
|
123
|
-
with workspace.app_locks((app.host, app.id) for app in apps):
|
|
124
|
-
ret = asyncio.run(_apush(server, uploads))
|
|
125
|
-
if ret == 0:
|
|
126
|
-
# re-pickle the manifests so they keep tracking what the
|
|
127
|
-
# server holds - the baseline 'vortex pull' diffs against
|
|
128
|
-
for app in apps:
|
|
129
|
-
workspace.mkdir(app)
|
|
130
|
-
# A refused element (nested classes) is a failure even when every
|
|
131
|
-
# other upload succeeded - scripts and agents must see it
|
|
132
|
-
return 1 if any_refused else ret
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|