systemlink-cli 3.0.0__py3-none-any.whl → 3.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slcli/_version.py +1 -1
- slcli/migrate_click.py +18 -9
- slcli/migration/command.py +92 -43
- slcli/migration/flows/discover.py +10 -2
- slcli/migration/flows/migrate.py +53 -0
- slcli/migration/orchestration.py +38 -14
- slcli/migration/schema.py +1 -1
- slcli/migration/state.py +3 -0
- slcli/migration/testmonitor/client.py +1 -1
- slcli/migration/testmonitor/discovery.py +7 -3
- slcli/migration/testmonitor/discovery_task.py +8 -11
- slcli/migration/workspace_task.py +65 -0
- slcli/migration/workspaces.py +114 -0
- {systemlink_cli-3.0.0.dist-info → systemlink_cli-3.0.2.dist-info}/METADATA +1 -1
- {systemlink_cli-3.0.0.dist-info → systemlink_cli-3.0.2.dist-info}/RECORD +18 -15
- {systemlink_cli-3.0.0.dist-info → systemlink_cli-3.0.2.dist-info}/WHEEL +0 -0
- {systemlink_cli-3.0.0.dist-info → systemlink_cli-3.0.2.dist-info}/entry_points.txt +0 -0
- {systemlink_cli-3.0.0.dist-info → systemlink_cli-3.0.2.dist-info}/licenses/LICENSE +0 -0
slcli/_version.py
CHANGED
slcli/migrate_click.py
CHANGED
|
@@ -86,13 +86,18 @@ def register_migrate_command(cli: click.Group) -> None:
|
|
|
86
86
|
|
|
87
87
|
@cli.command(name="migrate", hidden=not migration_extra_installed())
|
|
88
88
|
@click.option("--source-profile", help="Named source SystemLink profile for a new migration.")
|
|
89
|
+
@click.option(
|
|
90
|
+
"--destination-profile",
|
|
91
|
+
help="Named destination SystemLink profile for a new migration.",
|
|
92
|
+
)
|
|
89
93
|
@click.option(
|
|
90
94
|
"--workspace-mappings",
|
|
91
95
|
callback=_parse_workspace_mappings,
|
|
92
96
|
help=(
|
|
93
97
|
"Comma-delimited SOURCE or SOURCE:DESTINATION workspace mappings. "
|
|
94
98
|
"Escape ',' or ':' in a workspace name with a backslash, for example Lab\\:A:Dest. "
|
|
95
|
-
"Only the listed source workspaces are migrated; all when omitted."
|
|
99
|
+
"Only the listed source workspaces are migrated; all when omitted. "
|
|
100
|
+
"Destination workspaces must already exist."
|
|
96
101
|
),
|
|
97
102
|
)
|
|
98
103
|
@click.option("--migration-id", help="Repeat discovery for an existing migration.")
|
|
@@ -103,20 +108,22 @@ def register_migrate_command(cli: click.Group) -> None:
|
|
|
103
108
|
)
|
|
104
109
|
def migrate(
|
|
105
110
|
source_profile: Optional[str],
|
|
111
|
+
destination_profile: Optional[str],
|
|
106
112
|
workspace_mappings: Optional[Dict[str, str]],
|
|
107
113
|
migration_id: Optional[str],
|
|
108
114
|
prefect_api_url: Optional[str],
|
|
109
115
|
) -> None:
|
|
110
116
|
"""Discover and migrate resources from a SystemLink Server."""
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
):
|
|
117
|
+
new_run_options = (source_profile, destination_profile, workspace_mappings)
|
|
118
|
+
if migration_id is not None and any(value is not None for value in new_run_options):
|
|
114
119
|
raise click.UsageError(
|
|
115
|
-
"--migration-id cannot be combined with --source-profile
|
|
120
|
+
"--migration-id cannot be combined with --source-profile, "
|
|
121
|
+
"--destination-profile, or --workspace-mappings"
|
|
116
122
|
)
|
|
117
|
-
if migration_id is None and source_profile is None:
|
|
123
|
+
if migration_id is None and (source_profile is None or destination_profile is None):
|
|
118
124
|
raise click.UsageError(
|
|
119
|
-
"Provide --source-profile for a new migration
|
|
125
|
+
"Provide --source-profile and --destination-profile for a new migration "
|
|
126
|
+
"or --migration-id for an existing one"
|
|
120
127
|
)
|
|
121
128
|
|
|
122
129
|
try:
|
|
@@ -131,5 +138,7 @@ def register_migrate_command(cli: click.Group) -> None:
|
|
|
131
138
|
|
|
132
139
|
if migration_id is not None:
|
|
133
140
|
command.resume_migration(migration_id, prefect_api_url)
|
|
134
|
-
elif source_profile is not None:
|
|
135
|
-
command.start_migration(
|
|
141
|
+
elif source_profile is not None and destination_profile is not None:
|
|
142
|
+
command.start_migration(
|
|
143
|
+
source_profile, destination_profile, workspace_mappings, prefect_api_url
|
|
144
|
+
)
|
slcli/migration/command.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Compose a
|
|
1
|
+
"""Compose a migration run for the ``migrate`` command."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
from collections.abc import Generator
|
|
@@ -8,13 +8,20 @@ from typing import Dict, Optional
|
|
|
8
8
|
from urllib.parse import urlparse
|
|
9
9
|
|
|
10
10
|
import click
|
|
11
|
+
import requests
|
|
11
12
|
|
|
12
13
|
from slcli._version import __version__
|
|
13
14
|
from slcli.migration.connection import MigrationConnection, resolve_migration_connection
|
|
14
|
-
from slcli.migration.orchestration import prefect_server,
|
|
15
|
-
from slcli.migration.state import
|
|
15
|
+
from slcli.migration.orchestration import prefect_server, run_migration
|
|
16
|
+
from slcli.migration.state import (
|
|
17
|
+
MigrationRun,
|
|
18
|
+
RunMetadata,
|
|
19
|
+
create_migration_run,
|
|
20
|
+
open_migration_run,
|
|
21
|
+
)
|
|
16
22
|
from slcli.migration.testmonitor.models import TestMonitorDiscovery
|
|
17
|
-
from slcli.migration.
|
|
23
|
+
from slcli.migration.workspaces import WorkspaceMapping, WorkspaceMappingError
|
|
24
|
+
from slcli.utils import handle_api_error
|
|
18
25
|
|
|
19
26
|
_logger = logging.getLogger("slcli.migration")
|
|
20
27
|
MIGRATION_LOG = "migration.log"
|
|
@@ -22,16 +29,17 @@ MIGRATION_LOG = "migration.log"
|
|
|
22
29
|
|
|
23
30
|
def start_migration(
|
|
24
31
|
source_profile: str,
|
|
32
|
+
destination_profile: str,
|
|
25
33
|
workspace_mappings: Optional[Dict[str, str]],
|
|
26
34
|
prefect_api_url: Optional[str],
|
|
27
35
|
) -> None:
|
|
28
|
-
"""Create a new migration run
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
36
|
+
"""Create a new migration run, then discover resources and synchronize workspaces."""
|
|
37
|
+
source = resolve_migration_connection(source_profile)
|
|
38
|
+
destination = resolve_migration_connection(destination_profile)
|
|
39
|
+
_echo_connections(source, destination)
|
|
40
|
+
run = create_migration_run(source_profile, destination_profile, workspace_mappings)
|
|
41
|
+
_, metadata = open_migration_run(run.migration_id, run.directory.parent)
|
|
42
|
+
_migrate(run, metadata, source, destination, prefect_api_url)
|
|
35
43
|
|
|
36
44
|
|
|
37
45
|
def resume_migration(migration_id: str, prefect_api_url: Optional[str]) -> None:
|
|
@@ -40,9 +48,10 @@ def resume_migration(migration_id: str, prefect_api_url: Optional[str]) -> None:
|
|
|
40
48
|
run, metadata = open_migration_run(migration_id)
|
|
41
49
|
except ValueError as exc:
|
|
42
50
|
raise click.ClickException(str(exc)) from exc
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
51
|
+
source = resolve_migration_connection(metadata.source_profile)
|
|
52
|
+
destination = resolve_migration_connection(metadata.destination_profile)
|
|
53
|
+
_echo_connections(source, destination)
|
|
54
|
+
_migrate(run, metadata, source, destination, prefect_api_url)
|
|
46
55
|
|
|
47
56
|
|
|
48
57
|
def format_summary(discovery: TestMonitorDiscovery) -> str:
|
|
@@ -63,24 +72,41 @@ def _all_workspaces_failed(discovery: TestMonitorDiscovery) -> bool:
|
|
|
63
72
|
return bool(discovery.workspaces) and len(discovery.failures) == len(discovery.workspaces)
|
|
64
73
|
|
|
65
74
|
|
|
66
|
-
def
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
75
|
+
def format_workspace_mapping(mapping: WorkspaceMapping) -> str:
|
|
76
|
+
"""Return each source workspace and the destination workspace it migrates to."""
|
|
77
|
+
return "\n".join(
|
|
78
|
+
[
|
|
79
|
+
"Workspace mapping:",
|
|
80
|
+
*(
|
|
81
|
+
f" {workspace.source_name} -> {workspace.destination_name}"
|
|
82
|
+
for workspace in mapping.workspaces
|
|
83
|
+
),
|
|
84
|
+
]
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _echo_connections(source: MigrationConnection, destination: MigrationConnection) -> None:
|
|
89
|
+
for role, connection in (("Source", source), ("Destination", destination)):
|
|
90
|
+
click.echo(f"{role}: {connection.profile_name} ({connection.base_url})")
|
|
91
|
+
if urlparse(connection.base_url).scheme.lower() != "https":
|
|
92
|
+
click.echo(f"Warning: {role.lower()} connection does not use HTTPS.", err=True)
|
|
93
|
+
if connection.ssl_verify is False:
|
|
94
|
+
click.echo(
|
|
95
|
+
f"Warning: {role.lower()} TLS certificate verification is disabled.", err=True
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _mapping_error(
|
|
100
|
+
exc: WorkspaceMappingError, workspace_mappings: Optional[Dict[str, str]]
|
|
101
|
+
) -> click.ClickException:
|
|
102
|
+
# TODO(AB#4088608): Synchronize workspaces will create missing destination workspaces.
|
|
103
|
+
hint = (
|
|
104
|
+
"Destination workspaces are not created by migration."
|
|
105
|
+
if workspace_mappings is not None
|
|
106
|
+
else "Create the destination workspaces or select workspaces with "
|
|
107
|
+
"--workspace-mappings. Destination workspaces are not created by migration."
|
|
108
|
+
)
|
|
109
|
+
return click.ClickException(f"{exc}. {hint}")
|
|
84
110
|
|
|
85
111
|
|
|
86
112
|
@contextmanager
|
|
@@ -99,42 +125,65 @@ def _run_log(log_path: Path) -> Generator[None, None, None]:
|
|
|
99
125
|
handler.close()
|
|
100
126
|
|
|
101
127
|
|
|
102
|
-
def
|
|
128
|
+
def _migrate(
|
|
103
129
|
run: MigrationRun,
|
|
104
|
-
|
|
105
|
-
|
|
130
|
+
metadata: RunMetadata,
|
|
131
|
+
source: MigrationConnection,
|
|
132
|
+
destination: MigrationConnection,
|
|
106
133
|
prefect_api_url: Optional[str],
|
|
107
134
|
) -> None:
|
|
108
135
|
click.echo(f"Migration ID: {run.migration_id}")
|
|
109
136
|
log_path = run.directory / MIGRATION_LOG
|
|
137
|
+
workspace_mappings = metadata.workspace_mappings
|
|
110
138
|
workspaces = list(workspace_mappings) if workspace_mappings is not None else None
|
|
111
139
|
server_mode = "managed" if prefect_api_url is None else "external"
|
|
112
140
|
with _run_log(log_path):
|
|
113
141
|
_logger.info(
|
|
114
|
-
"
|
|
115
|
-
"prefect_server=%s",
|
|
142
|
+
"Migration started: migration=%s cli=%s profile=%s endpoint=%s workspaces=%s "
|
|
143
|
+
"destination_profile=%s destination_endpoint=%s prefect_server=%s",
|
|
116
144
|
run.migration_id,
|
|
117
145
|
__version__,
|
|
118
|
-
|
|
119
|
-
|
|
146
|
+
source.profile_name,
|
|
147
|
+
source.base_url,
|
|
120
148
|
workspaces if workspaces is not None else "all",
|
|
149
|
+
destination.profile_name,
|
|
150
|
+
destination.base_url,
|
|
121
151
|
server_mode,
|
|
122
152
|
)
|
|
123
153
|
try:
|
|
124
154
|
with prefect_server(run, prefect_api_url) as api_url:
|
|
125
|
-
|
|
155
|
+
outcome = run_migration(run, metadata, api_url)
|
|
156
|
+
except WorkspaceMappingError as exc:
|
|
157
|
+
_log_failure(exc, log_path)
|
|
158
|
+
raise _mapping_error(exc, workspace_mappings) from exc
|
|
159
|
+
except requests.RequestException as exc:
|
|
160
|
+
_log_failure(exc, log_path)
|
|
161
|
+
handle_api_error(exc)
|
|
162
|
+
raise
|
|
126
163
|
except BaseException as exc:
|
|
127
|
-
|
|
128
|
-
click.echo(f"Log: {log_path}", err=True)
|
|
164
|
+
_log_failure(exc, log_path)
|
|
129
165
|
raise
|
|
130
|
-
|
|
166
|
+
_logger.info(
|
|
167
|
+
"Workspace mapping: %s",
|
|
168
|
+
[
|
|
169
|
+
f"{w.source_name}({w.source_id}) -> {w.destination_name}({w.destination_id})"
|
|
170
|
+
for w in outcome.mapping.workspaces
|
|
171
|
+
],
|
|
172
|
+
)
|
|
173
|
+
test_monitor = outcome.discovery.test_monitor
|
|
131
174
|
_logger.info(
|
|
132
175
|
"Discovery completed: products=%d results=%d failed_workspaces=%s",
|
|
133
176
|
len(test_monitor.products),
|
|
134
177
|
len(test_monitor.results),
|
|
135
178
|
[failure.workspace for failure in test_monitor.failures],
|
|
136
179
|
)
|
|
180
|
+
click.echo(format_workspace_mapping(outcome.mapping))
|
|
137
181
|
click.echo(format_summary(test_monitor))
|
|
138
182
|
click.echo(f"Log: {log_path}")
|
|
139
183
|
if _all_workspaces_failed(test_monitor):
|
|
140
184
|
raise click.ClickException("Discovery failed for every workspace.")
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _log_failure(exc: BaseException, log_path: Path) -> None:
|
|
188
|
+
_logger.error("Migration failed: %s: %s", type(exc).__name__, exc)
|
|
189
|
+
click.echo(f"Log: {log_path}", err=True)
|
|
@@ -8,6 +8,8 @@ from pydantic import ConfigDict
|
|
|
8
8
|
from slcli.migration.schema import VersionedModel
|
|
9
9
|
from slcli.migration.testmonitor.discovery_task import discover_test_monitor_task
|
|
10
10
|
from slcli.migration.testmonitor.models import TestMonitorDiscovery
|
|
11
|
+
from slcli.migration.workspace_task import discover_workspaces_task
|
|
12
|
+
from slcli.migration.workspaces import DiscoveredWorkspaces
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class MigrationDiscovery(VersionedModel):
|
|
@@ -15,6 +17,7 @@ class MigrationDiscovery(VersionedModel):
|
|
|
15
17
|
|
|
16
18
|
model_config = ConfigDict(frozen=True)
|
|
17
19
|
|
|
20
|
+
workspaces: DiscoveredWorkspaces
|
|
18
21
|
test_monitor: TestMonitorDiscovery
|
|
19
22
|
|
|
20
23
|
|
|
@@ -29,13 +32,18 @@ def discover_flow(
|
|
|
29
32
|
source_profile: str,
|
|
30
33
|
workspaces: Optional[List[str]],
|
|
31
34
|
) -> MigrationDiscovery:
|
|
32
|
-
"""
|
|
35
|
+
"""Discover source workspaces, then each supported resource type within them.
|
|
33
36
|
|
|
34
37
|
Args:
|
|
35
38
|
migration_id: Migration that scopes task cache keys.
|
|
36
39
|
source_profile: Name of the source profile.
|
|
37
40
|
workspaces: Source workspace names to discover; all workspaces when None.
|
|
41
|
+
|
|
42
|
+
Raises:
|
|
43
|
+
WorkspaceMappingError: If a requested workspace does not exist on the source.
|
|
38
44
|
"""
|
|
45
|
+
discovered = discover_workspaces_task(migration_id, source_profile, workspaces)
|
|
39
46
|
return MigrationDiscovery(
|
|
40
|
-
|
|
47
|
+
workspaces=discovered,
|
|
48
|
+
test_monitor=discover_test_monitor_task(migration_id, source_profile, discovered),
|
|
41
49
|
)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Migration flow: discover, then synchronize workspaces."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict, List, Optional
|
|
4
|
+
|
|
5
|
+
from prefect import flow
|
|
6
|
+
from pydantic import ConfigDict
|
|
7
|
+
|
|
8
|
+
from slcli.migration.flows.discover import MigrationDiscovery, discover_flow
|
|
9
|
+
from slcli.migration.schema import VersionedModel
|
|
10
|
+
from slcli.migration.workspace_task import synchronize_workspaces_task
|
|
11
|
+
from slcli.migration.workspaces import WorkspaceMapping
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MigrationOutcome(VersionedModel):
|
|
15
|
+
"""Discovery and workspace mapping of one run."""
|
|
16
|
+
|
|
17
|
+
model_config = ConfigDict(frozen=True)
|
|
18
|
+
|
|
19
|
+
discovery: MigrationDiscovery
|
|
20
|
+
mapping: WorkspaceMapping
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@flow(
|
|
24
|
+
name="systemlink-migration",
|
|
25
|
+
flow_run_name="migration-{migration_id}",
|
|
26
|
+
persist_result=True,
|
|
27
|
+
result_serializer="json",
|
|
28
|
+
)
|
|
29
|
+
def migrate_flow(
|
|
30
|
+
migration_id: str,
|
|
31
|
+
source_profile: str,
|
|
32
|
+
destination_profile: str,
|
|
33
|
+
workspace_mappings: Optional[Dict[str, str]],
|
|
34
|
+
) -> MigrationOutcome:
|
|
35
|
+
"""Discover source resources, then synchronize workspaces to the destination.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
migration_id: Migration that scopes task cache keys.
|
|
39
|
+
source_profile: Name of the source profile.
|
|
40
|
+
destination_profile: Name of the destination profile.
|
|
41
|
+
workspace_mappings: Source to destination workspace names; all by name when None.
|
|
42
|
+
|
|
43
|
+
Raises:
|
|
44
|
+
WorkspaceMappingError: If a source or destination workspace does not exist.
|
|
45
|
+
"""
|
|
46
|
+
workspaces: Optional[List[str]] = (
|
|
47
|
+
list(workspace_mappings) if workspace_mappings is not None else None
|
|
48
|
+
)
|
|
49
|
+
discovery = discover_flow(migration_id, source_profile, workspaces)
|
|
50
|
+
mapping = synchronize_workspaces_task(
|
|
51
|
+
migration_id, destination_profile, discovery.workspaces, workspace_mappings
|
|
52
|
+
)
|
|
53
|
+
return MigrationOutcome(discovery=discovery, mapping=mapping)
|
slcli/migration/orchestration.py
CHANGED
|
@@ -13,8 +13,9 @@ from prefect.settings import (
|
|
|
13
13
|
)
|
|
14
14
|
|
|
15
15
|
from slcli.migration.flows.discover import MigrationDiscovery, discover_flow
|
|
16
|
+
from slcli.migration.flows.migrate import MigrationOutcome, migrate_flow
|
|
16
17
|
from slcli.migration.prefect_server import ManagedPrefectServer
|
|
17
|
-
from slcli.migration.state import MigrationRun
|
|
18
|
+
from slcli.migration.state import MigrationRun, RunMetadata
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
@contextmanager
|
|
@@ -33,6 +34,23 @@ def prefect_server(run: MigrationRun, api_url: Optional[str] = None) -> Generato
|
|
|
33
34
|
yield managed_api_url
|
|
34
35
|
|
|
35
36
|
|
|
37
|
+
@contextmanager
|
|
38
|
+
def _run_settings(run: MigrationRun, api_url: str) -> Generator[None, None, None]:
|
|
39
|
+
with temporary_settings(
|
|
40
|
+
updates={
|
|
41
|
+
PREFECT_HOME: run.client_home,
|
|
42
|
+
PREFECT_API_URL: api_url,
|
|
43
|
+
PREFECT_LOGGING_TO_API_ENABLED: False,
|
|
44
|
+
"results.local_storage_path": run.results_path,
|
|
45
|
+
}
|
|
46
|
+
):
|
|
47
|
+
try:
|
|
48
|
+
yield
|
|
49
|
+
finally:
|
|
50
|
+
# Deliver queued task-state events before the server may be stopped.
|
|
51
|
+
EventsWorker.drain_all()
|
|
52
|
+
|
|
53
|
+
|
|
36
54
|
def run_discovery(
|
|
37
55
|
run: MigrationRun,
|
|
38
56
|
source_profile: str,
|
|
@@ -47,16 +65,22 @@ def run_discovery(
|
|
|
47
65
|
workspaces: Source workspace names to discover; all workspaces when None.
|
|
48
66
|
api_url: Prefect API URL from :func:`prefect_server`.
|
|
49
67
|
"""
|
|
50
|
-
with
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
68
|
+
with _run_settings(run, api_url):
|
|
69
|
+
return discover_flow(run.migration_id, source_profile, workspaces)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def run_migration(run: MigrationRun, metadata: RunMetadata, api_url: str) -> MigrationOutcome:
|
|
73
|
+
"""Run the migration flow for a run on a Prefect server.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
run: Migration run that provides the client home and result storage.
|
|
77
|
+
metadata: Persisted run inputs, including profiles and workspace mappings.
|
|
78
|
+
api_url: Prefect API URL from :func:`prefect_server`.
|
|
79
|
+
"""
|
|
80
|
+
with _run_settings(run, api_url):
|
|
81
|
+
return migrate_flow(
|
|
82
|
+
run.migration_id,
|
|
83
|
+
metadata.source_profile,
|
|
84
|
+
metadata.destination_profile,
|
|
85
|
+
metadata.workspace_mappings,
|
|
86
|
+
)
|
slcli/migration/schema.py
CHANGED
slcli/migration/state.py
CHANGED
|
@@ -28,6 +28,7 @@ class RunMetadata(VersionedModel):
|
|
|
28
28
|
migration_id: str
|
|
29
29
|
cli_version: str = __version__
|
|
30
30
|
source_profile: str
|
|
31
|
+
destination_profile: str
|
|
31
32
|
workspace_mappings: Optional[Dict[str, str]]
|
|
32
33
|
created_at: datetime
|
|
33
34
|
|
|
@@ -91,6 +92,7 @@ def _validate_created_migration_id(migration_id: str) -> None:
|
|
|
91
92
|
|
|
92
93
|
def create_migration_run(
|
|
93
94
|
source_profile: str,
|
|
95
|
+
destination_profile: str,
|
|
94
96
|
workspace_mappings: Optional[Dict[str, str]],
|
|
95
97
|
root: Optional[Path] = None,
|
|
96
98
|
) -> MigrationRun:
|
|
@@ -106,6 +108,7 @@ def create_migration_run(
|
|
|
106
108
|
RunMetadata(
|
|
107
109
|
migration_id=migration_id,
|
|
108
110
|
source_profile=source_profile,
|
|
111
|
+
destination_profile=destination_profile,
|
|
109
112
|
workspace_mappings=workspace_mappings,
|
|
110
113
|
created_at=datetime.now(timezone.utc),
|
|
111
114
|
)
|
|
@@ -113,8 +113,10 @@ def discover_test_monitor(
|
|
|
113
113
|
|
|
114
114
|
Only selected workspaces are queried, so product ownership is inferred from
|
|
115
115
|
their results alone. A failed workspace is reported and excluded from that
|
|
116
|
-
inference. Products with no results
|
|
117
|
-
|
|
116
|
+
inference. Products with no results belong to Default, so they are included
|
|
117
|
+
only when every workspace, Default among them, is discovered without failures.
|
|
118
|
+
A failed workspace could hide their results. Every discovered item's workspace
|
|
119
|
+
is therefore a selected workspace.
|
|
118
120
|
|
|
119
121
|
Args:
|
|
120
122
|
client: Test Monitor queries for the source server.
|
|
@@ -129,7 +131,9 @@ def discover_test_monitor(
|
|
|
129
131
|
scan = _scan_results(
|
|
130
132
|
client, {name: id_ for name, id_ in all_workspaces.items() if name in selected}
|
|
131
133
|
)
|
|
132
|
-
migrate_products_with_no_results =
|
|
134
|
+
migrate_products_with_no_results = (
|
|
135
|
+
selected == set(all_workspaces) and DEFAULT_WORKSPACE in selected and not scan.failures
|
|
136
|
+
)
|
|
133
137
|
products: List[DiscoveredProduct] = []
|
|
134
138
|
product_ids_by_part_number: Dict[str, str] = {}
|
|
135
139
|
for product in client.query_products():
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
"""Persisted Prefect task for Test Monitor discovery."""
|
|
2
2
|
|
|
3
|
-
from typing import List, Optional
|
|
4
|
-
|
|
5
3
|
from prefect import task
|
|
6
4
|
from prefect.cache_policies import INPUTS
|
|
7
5
|
|
|
@@ -9,7 +7,7 @@ from slcli.migration.connection import resolve_migration_connection
|
|
|
9
7
|
from slcli.migration.testmonitor.client import TestMonitorClient
|
|
10
8
|
from slcli.migration.testmonitor.discovery import discover_test_monitor
|
|
11
9
|
from slcli.migration.testmonitor.models import TestMonitorDiscovery
|
|
12
|
-
from slcli.migration.
|
|
10
|
+
from slcli.migration.workspaces import DiscoveredWorkspaces
|
|
13
11
|
|
|
14
12
|
|
|
15
13
|
@task(
|
|
@@ -22,17 +20,16 @@ from slcli.migration.workspace_client import WorkspaceClient
|
|
|
22
20
|
def discover_test_monitor_task(
|
|
23
21
|
migration_id: str,
|
|
24
22
|
source_profile: str,
|
|
25
|
-
workspaces:
|
|
23
|
+
workspaces: DiscoveredWorkspaces,
|
|
26
24
|
) -> TestMonitorDiscovery:
|
|
27
|
-
"""Discover Test Monitor resources
|
|
25
|
+
"""Discover Test Monitor resources in the discovered source workspaces.
|
|
28
26
|
|
|
29
27
|
Inputs are references, never credentials. ``migration_id`` is unused by the
|
|
30
28
|
body but scopes the cache key to one migration.
|
|
31
29
|
"""
|
|
32
30
|
connection = resolve_migration_connection(source_profile)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
workspace.name: workspace.id
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
return discover_test_monitor(TestMonitorClient(connection, session), all_workspaces, workspaces)
|
|
31
|
+
return discover_test_monitor(
|
|
32
|
+
TestMonitorClient(connection),
|
|
33
|
+
{workspace.name: workspace.id for workspace in workspaces.source},
|
|
34
|
+
workspaces.selected,
|
|
35
|
+
)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Persisted Prefect tasks for migration workspaces."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict, List, Optional
|
|
4
|
+
|
|
5
|
+
from prefect import task
|
|
6
|
+
from prefect.cache_policies import INPUTS
|
|
7
|
+
|
|
8
|
+
from slcli.migration.connection import resolve_migration_connection
|
|
9
|
+
from slcli.migration.workspace_client import WorkspaceClient
|
|
10
|
+
from slcli.migration.workspaces import (
|
|
11
|
+
DiscoveredWorkspaces,
|
|
12
|
+
WorkspaceMapping,
|
|
13
|
+
discover_workspaces,
|
|
14
|
+
synchronize_workspaces,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@task(
|
|
19
|
+
name="discover-workspaces",
|
|
20
|
+
persist_result=True,
|
|
21
|
+
result_serializer="json",
|
|
22
|
+
cache_result_in_memory=False,
|
|
23
|
+
cache_policy=INPUTS,
|
|
24
|
+
)
|
|
25
|
+
def discover_workspaces_task(
|
|
26
|
+
migration_id: str, source_profile: str, workspaces: Optional[List[str]]
|
|
27
|
+
) -> DiscoveredWorkspaces:
|
|
28
|
+
"""Discover the source workspaces in scope of a migration.
|
|
29
|
+
|
|
30
|
+
``migration_id`` is unused by the body but scopes the cache key to one migration.
|
|
31
|
+
|
|
32
|
+
Raises:
|
|
33
|
+
WorkspaceMappingError: If a requested workspace does not exist on the source.
|
|
34
|
+
"""
|
|
35
|
+
source = WorkspaceClient(resolve_migration_connection(source_profile)).query_workspaces()
|
|
36
|
+
return discover_workspaces(source, workspaces)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# Later migration tasks are cached by the mapping, so a resumed run must reuse
|
|
40
|
+
# the mapping it synchronized first. A synchronization that raised is not cached.
|
|
41
|
+
@task(
|
|
42
|
+
name="synchronize-workspaces",
|
|
43
|
+
persist_result=True,
|
|
44
|
+
result_serializer="json",
|
|
45
|
+
cache_result_in_memory=False,
|
|
46
|
+
cache_policy=INPUTS,
|
|
47
|
+
)
|
|
48
|
+
def synchronize_workspaces_task(
|
|
49
|
+
migration_id: str,
|
|
50
|
+
destination_profile: str,
|
|
51
|
+
discovered: DiscoveredWorkspaces,
|
|
52
|
+
workspace_mappings: Optional[Dict[str, str]],
|
|
53
|
+
) -> WorkspaceMapping:
|
|
54
|
+
"""Map discovered source workspaces to destination workspaces.
|
|
55
|
+
|
|
56
|
+
``migration_id`` is unused by the body but scopes the cache key to one migration.
|
|
57
|
+
|
|
58
|
+
Raises:
|
|
59
|
+
WorkspaceMappingError: If a destination workspace does not exist.
|
|
60
|
+
"""
|
|
61
|
+
# TODO(AB#4088608): Create missing destination workspaces.
|
|
62
|
+
destination = WorkspaceClient(
|
|
63
|
+
resolve_migration_connection(destination_profile)
|
|
64
|
+
).query_workspaces()
|
|
65
|
+
return synchronize_workspaces(discovered, destination, workspace_mappings)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Discover source workspaces and synchronize them to destination workspaces."""
|
|
2
|
+
|
|
3
|
+
from typing import Iterable, Mapping, Optional, Sequence, Tuple
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict
|
|
6
|
+
|
|
7
|
+
from slcli.migration.workspace_client import Workspace
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DiscoveredWorkspaces(BaseModel):
|
|
11
|
+
"""Every accessible source workspace and the names in scope of a migration."""
|
|
12
|
+
|
|
13
|
+
model_config = ConfigDict(frozen=True)
|
|
14
|
+
|
|
15
|
+
source: Tuple[Workspace, ...] = ()
|
|
16
|
+
selected: Tuple[str, ...] = ()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class MappedWorkspace(BaseModel):
|
|
20
|
+
"""One source workspace and the existing destination workspace it migrates to."""
|
|
21
|
+
|
|
22
|
+
model_config = ConfigDict(frozen=True)
|
|
23
|
+
|
|
24
|
+
source_name: str
|
|
25
|
+
source_id: str
|
|
26
|
+
destination_name: str
|
|
27
|
+
destination_id: str
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class WorkspaceMapping(BaseModel):
|
|
31
|
+
"""Resolved source-to-destination workspaces, ordered by source name."""
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(frozen=True)
|
|
34
|
+
|
|
35
|
+
workspaces: Tuple[MappedWorkspace, ...] = ()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class WorkspaceMappingError(ValueError):
|
|
39
|
+
"""Requested workspaces that do not exist on the source or destination."""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self, missing_source: Sequence[str] = (), missing_destination: Sequence[str] = ()
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Record the missing workspaces on each side."""
|
|
45
|
+
self.missing_source = list(missing_source)
|
|
46
|
+
self.missing_destination = list(missing_destination)
|
|
47
|
+
problems = []
|
|
48
|
+
if missing_source:
|
|
49
|
+
problems.append(f"unknown source workspace: {', '.join(missing_source)}")
|
|
50
|
+
if missing_destination:
|
|
51
|
+
problems.append(f"missing destination workspace: {', '.join(missing_destination)}")
|
|
52
|
+
super().__init__("; ".join(problems))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def discover_workspaces(
|
|
56
|
+
source: Iterable[Workspace], requested: Optional[Iterable[str]]
|
|
57
|
+
) -> DiscoveredWorkspaces:
|
|
58
|
+
"""Select the source workspaces in scope of a migration.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
source: Every accessible source workspace.
|
|
62
|
+
requested: Source workspace names to migrate; all workspaces when None.
|
|
63
|
+
|
|
64
|
+
Raises:
|
|
65
|
+
WorkspaceMappingError: If a requested workspace does not exist on the source.
|
|
66
|
+
"""
|
|
67
|
+
workspaces = tuple(sorted(source, key=lambda workspace: workspace.name))
|
|
68
|
+
names = [workspace.name for workspace in workspaces]
|
|
69
|
+
selected = sorted(requested) if requested is not None else names
|
|
70
|
+
missing = sorted(set(selected).difference(names))
|
|
71
|
+
if missing:
|
|
72
|
+
raise WorkspaceMappingError(missing_source=missing)
|
|
73
|
+
return DiscoveredWorkspaces(source=workspaces, selected=tuple(selected))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def synchronize_workspaces(
|
|
77
|
+
discovered: DiscoveredWorkspaces,
|
|
78
|
+
destination: Iterable[Workspace],
|
|
79
|
+
requested: Optional[Mapping[str, str]],
|
|
80
|
+
) -> WorkspaceMapping:
|
|
81
|
+
"""Map discovered source workspaces to existing destination workspaces.
|
|
82
|
+
|
|
83
|
+
Destination workspaces are not created yet. Every selected source workspace
|
|
84
|
+
maps to the same-named destination workspace unless ``requested`` renames it.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
discovered: Source workspaces in scope of the migration.
|
|
88
|
+
destination: Every accessible destination workspace.
|
|
89
|
+
requested: Source to destination workspace names; all by name when None.
|
|
90
|
+
|
|
91
|
+
Raises:
|
|
92
|
+
WorkspaceMappingError: If a destination workspace does not exist.
|
|
93
|
+
"""
|
|
94
|
+
source_ids = {workspace.name: workspace.id for workspace in discovered.source}
|
|
95
|
+
destination_ids = {workspace.name: workspace.id for workspace in destination}
|
|
96
|
+
pairs = {name: (requested or {}).get(name, name) for name in discovered.selected}
|
|
97
|
+
missing = sorted(
|
|
98
|
+
f"{source_name} -> {destination_name}"
|
|
99
|
+
for source_name, destination_name in pairs.items()
|
|
100
|
+
if destination_name not in destination_ids
|
|
101
|
+
)
|
|
102
|
+
if missing:
|
|
103
|
+
raise WorkspaceMappingError(missing_destination=missing)
|
|
104
|
+
return WorkspaceMapping(
|
|
105
|
+
workspaces=tuple(
|
|
106
|
+
MappedWorkspace(
|
|
107
|
+
source_name=source_name,
|
|
108
|
+
source_id=source_ids[source_name],
|
|
109
|
+
destination_name=destination_name,
|
|
110
|
+
destination_id=destination_ids[destination_name],
|
|
111
|
+
)
|
|
112
|
+
for source_name, destination_name in pairs.items()
|
|
113
|
+
)
|
|
114
|
+
)
|
|
@@ -2,7 +2,7 @@ dff-editor/editor.js,sha256=VQJNqtkFSeXec_hidYdbsvhcKo8-dcdMoU5qvGVn3YQ,97488
|
|
|
2
2
|
dff-editor/index.html,sha256=Bbicid_Fx66W7ypHgSGJrUtkAEFcLP3HkRoxiGKZjCs,11657
|
|
3
3
|
slcli/__init__.py,sha256=ZRF050VsFgb4zeyyQzQINQ4eHFwEBBoUq-CPyziIB6g,66
|
|
4
4
|
slcli/__main__.py,sha256=wUlBlvYS3Tn4ipw9_7zMr2d1houRpNnALG0TxHg6xZg,829
|
|
5
|
-
slcli/_version.py,sha256=
|
|
5
|
+
slcli/_version.py,sha256=ZYtquDRVnpCWVkTfEGLId5PeiQSt9WT_4N26uKnabng,113
|
|
6
6
|
slcli/alarm_click.py,sha256=d5pobCgGr4J5AL0H-Tc8KZLyKkZWNeK4fcDWuYmMqJE,32189
|
|
7
7
|
slcli/asset_click.py,sha256=WTxiP_XvN7GOQ249omU--gs2u23T4ddLVKvWwH3LBsQ,44692
|
|
8
8
|
slcli/cli_formatters.py,sha256=ShrU2RbnQUEKhJ6ahWuv8c3zw7C-yg0o20i1opq9gZk,7275
|
|
@@ -59,23 +59,26 @@ slcli/mcp_click.py,sha256=AZ2WJ_odc2yEmitk24qrozNLq-hb9K6A-a0w12t7mL0,9246
|
|
|
59
59
|
slcli/mcp_reachability.py,sha256=jn1RxtjLPCwABymZFotK-hmer80ZqcLKx9LD9x5g0dA,1908
|
|
60
60
|
slcli/mcp_server.py,sha256=AqWYku0bzTC2RCv-oNQL2zqpIE6CsoPAsgRmVIkqFk0,32896
|
|
61
61
|
slcli/mcp_skills.py,sha256=iCdujb-ttaRZyASmrZgkNP5JX4yh0hAFXS02doy6o8I,18437
|
|
62
|
-
slcli/migrate_click.py,sha256=
|
|
62
|
+
slcli/migrate_click.py,sha256=Mf97-bVbPlSL1etBojp3y5j2LopGEHQokfUc_XDVJv4,5687
|
|
63
63
|
slcli/migration/__init__.py,sha256=fy6Du8e5u29dF-ThismOYvowW4KkqLgHgUxp9PQP8JM,64
|
|
64
64
|
slcli/migration/api.py,sha256=4be7GHH-PzuLD1JlKH-3c-6VjpgrQMmN8zp7i-qAeVY,399
|
|
65
|
-
slcli/migration/command.py,sha256=
|
|
65
|
+
slcli/migration/command.py,sha256=maJiRCJN9NcZsBgBb9FgGN4GMT74bakieZBosnin0rE,7477
|
|
66
66
|
slcli/migration/connection.py,sha256=tj4pdrzC-MqR-TOzyVDcv2OZIme-hRnDXTIIWuxwrhY,3892
|
|
67
67
|
slcli/migration/flows/__init__.py,sha256=pHYgu2DTzMazd69UXmmhTfX2vmY3sPReLZaDWdOL4-4,46
|
|
68
|
-
slcli/migration/flows/discover.py,sha256=
|
|
69
|
-
slcli/migration/
|
|
68
|
+
slcli/migration/flows/discover.py,sha256=hemiN3qtw9FRsiasliZety8dELpLFsmG7_jwLVOx4YI,1619
|
|
69
|
+
slcli/migration/flows/migrate.py,sha256=ucCAByjaUtlJkOiuEt4fgy9NAMpM0F26jmHLzSwE1uk,1791
|
|
70
|
+
slcli/migration/orchestration.py,sha256=-O2KVIpzPmS6huFRs2u5UxD3nDZlUrvFPwaJFyvsIaY,2927
|
|
70
71
|
slcli/migration/prefect_server.py,sha256=PaenJuXZC6gTF77A1KRZqGRFOha4q2NEsLttwXofWP4,7544
|
|
71
|
-
slcli/migration/schema.py,sha256=
|
|
72
|
-
slcli/migration/state.py,sha256=
|
|
72
|
+
slcli/migration/schema.py,sha256=L9lfo1JWrXN_xmvP08mSNECX4Lf6OKYduNJ4VfqhKNg,550
|
|
73
|
+
slcli/migration/state.py,sha256=DCE6nthX3TNhHSNXFe0myjbssPneJdV0prQEPuM3MS4,4434
|
|
73
74
|
slcli/migration/testmonitor/__init__.py,sha256=BuPHwwjotgw_4CXU-CTcYBwMO0VVJlTkL-I1uKOejlQ,38
|
|
74
|
-
slcli/migration/testmonitor/client.py,sha256=
|
|
75
|
-
slcli/migration/testmonitor/discovery.py,sha256=
|
|
76
|
-
slcli/migration/testmonitor/discovery_task.py,sha256=
|
|
75
|
+
slcli/migration/testmonitor/client.py,sha256=Z_mMI98ihiTBxBFaO-0sPXpEW8g6PoE7zNwNRMI-rM4,3085
|
|
76
|
+
slcli/migration/testmonitor/discovery.py,sha256=ZUd4RCY9d_0anwpUAo4MzmGDhIOqtON9akLPK-YDP6k,6223
|
|
77
|
+
slcli/migration/testmonitor/discovery_task.py,sha256=Sylh6UyH-opaDLsKR_XSiCpbTq9L8Ig5fWiH1XPPCI8,1225
|
|
77
78
|
slcli/migration/testmonitor/models.py,sha256=kR8STukD6IcSqlbQI-iDzA6YvODOVlIKc7mFKb5ZVUs,1187
|
|
78
79
|
slcli/migration/workspace_client.py,sha256=ur53BPIsdPw2QoXCfokMEVYA_XgGl5X8Eu7zHHRJNXE,1448
|
|
80
|
+
slcli/migration/workspace_task.py,sha256=rckyWsZy-yvCsURHs5RhRnIt7Zzoiy8Kng7yQvIq1R8,2158
|
|
81
|
+
slcli/migration/workspaces.py,sha256=S5Gf0_qHw7IxJqjme9cokv6PuUQcItLH9cwE7G4Znq0,4149
|
|
79
82
|
slcli/notebook_click.py,sha256=DnjSNqI6XS2Kzeojh6O0k_qQrcu1nLUaj0jaBYCAcSs,71646
|
|
80
83
|
slcli/paths.py,sha256=6buJ3dvvYFmHyj01W2d-p_bHDkYTYjJEZGe0CvykTdU,941
|
|
81
84
|
slcli/pkce.py,sha256=MMc4958hqZYDrx96zNG3-TGpZG2NXX5Og6Tl9IXYyBg,22355
|
|
@@ -211,8 +214,8 @@ slcli/workflows_click.py,sha256=0SP4gFOxJeTw4xpgO3F3FTmAp1D0uYNob_eWoRCCKnI,4069
|
|
|
211
214
|
slcli/workitem_click.py,sha256=UbomK7zO9FWNSXEx0UA_gMmJ_uNZfqyrekYSdvhwPTo,88940
|
|
212
215
|
slcli/workspace_click.py,sha256=K1z24ipJ2cOCuG7ygIlNWMiV_cwOguTDLqdxbr-kNxE,21036
|
|
213
216
|
slcli/workspace_utils.py,sha256=7QOtotmv3ghJs6a0fDQVojKxrBKPeq_GtE4F1SmZKnM,6872
|
|
214
|
-
systemlink_cli-3.0.
|
|
215
|
-
systemlink_cli-3.0.
|
|
216
|
-
systemlink_cli-3.0.
|
|
217
|
-
systemlink_cli-3.0.
|
|
218
|
-
systemlink_cli-3.0.
|
|
217
|
+
systemlink_cli-3.0.2.dist-info/METADATA,sha256=8N2SUIxkFtMSZMBfwXXiZqqhopA4eWMABAK_eO7as7o,1583
|
|
218
|
+
systemlink_cli-3.0.2.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
|
|
219
|
+
systemlink_cli-3.0.2.dist-info/entry_points.txt,sha256=V8IyRF-NYesTOTA_yB4-lVK1EsKZsSY5zOjzGbh5zdA,288
|
|
220
|
+
systemlink_cli-3.0.2.dist-info/licenses/LICENSE,sha256=ioLAEBMHywwQkOr14pBIc7nenmWrc-bgcMrqsB_Uvq8,1077
|
|
221
|
+
systemlink_cli-3.0.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|