furu 0.0.6__py3-none-any.whl → 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- furu/__init__.py +3 -15
- furu/aliases.py +53 -0
- furu/core/furu.py +153 -5
- furu/dashboard/api/models.py +18 -4
- furu/dashboard/api/routes.py +8 -0
- furu/dashboard/frontend/dist/assets/{index-BjyrY-Zz.js → index-NiDdQnqO.js} +15 -15
- furu/dashboard/frontend/dist/index.html +1 -1
- furu/dashboard/scanner.py +173 -147
- furu/migration.py +491 -763
- furu/schema.py +46 -0
- furu/storage/metadata.py +17 -1
- {furu-0.0.6.dist-info → furu-0.0.7.dist-info}/METADATA +1 -1
- {furu-0.0.6.dist-info → furu-0.0.7.dist-info}/RECORD +15 -14
- furu/migrate.py +0 -48
- {furu-0.0.6.dist-info → furu-0.0.7.dist-info}/WHEEL +0 -0
- {furu-0.0.6.dist-info → furu-0.0.7.dist-info}/entry_points.txt +0 -0
furu/migration.py
CHANGED
|
@@ -1,594 +1,389 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import datetime as _dt
|
|
4
|
-
import importlib
|
|
5
|
-
import json
|
|
6
|
-
import shutil
|
|
7
4
|
from collections.abc import Iterable, Mapping
|
|
8
5
|
from dataclasses import dataclass
|
|
9
6
|
from pathlib import Path
|
|
10
|
-
from typing import Literal,
|
|
7
|
+
from typing import Literal, Protocol, cast
|
|
11
8
|
|
|
12
9
|
import chz
|
|
13
10
|
from chz.util import MISSING as CHZ_MISSING, MISSING_TYPE
|
|
14
|
-
from chz.validators import for_all_fields, typecheck
|
|
15
11
|
|
|
12
|
+
from .aliases import collect_aliases
|
|
16
13
|
from .config import FURU_CONFIG
|
|
17
|
-
from .
|
|
18
|
-
|
|
14
|
+
from .schema import (
|
|
15
|
+
schema_key_from_cls,
|
|
16
|
+
schema_key_from_furu_obj,
|
|
17
|
+
schema_key_from_metadata_raw,
|
|
18
|
+
)
|
|
19
19
|
from .serialization import FuruSerializer
|
|
20
20
|
from .serialization.serializer import JsonValue
|
|
21
21
|
from .storage import MetadataManager, MigrationManager, MigrationRecord, StateManager
|
|
22
|
-
from .storage.
|
|
22
|
+
from .storage.migration import RootKind
|
|
23
|
+
from .storage.state import _StateResultMigrated
|
|
23
24
|
|
|
24
25
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
Primitive | Furu | tuple["MigrationValue", ...] | dict[str, "MigrationValue"]
|
|
28
|
-
)
|
|
26
|
+
class _FuruClass(Protocol):
|
|
27
|
+
version_controlled: bool
|
|
29
28
|
|
|
29
|
+
@classmethod
|
|
30
|
+
def _namespace(cls) -> Path: ...
|
|
30
31
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
from_namespace: str
|
|
34
|
-
to_namespace: str
|
|
32
|
+
|
|
33
|
+
MigrationConflict = Literal["throw", "skip"]
|
|
35
34
|
|
|
36
35
|
|
|
37
36
|
@dataclass(frozen=True)
|
|
38
37
|
class FuruRef:
|
|
39
38
|
namespace: str
|
|
40
39
|
furu_hash: str
|
|
41
|
-
root:
|
|
40
|
+
root: RootKind
|
|
42
41
|
directory: Path
|
|
43
42
|
|
|
44
43
|
|
|
45
|
-
@dataclass(frozen=True)
|
|
46
|
-
class MigrationCandidate:
|
|
47
|
-
from_ref: FuruRef
|
|
48
|
-
to_ref: FuruRef
|
|
49
|
-
to_namespace: str
|
|
50
|
-
to_config: dict[str, JsonValue]
|
|
51
|
-
defaults_applied: dict[str, MigrationValue]
|
|
52
|
-
fields_dropped: list[str]
|
|
53
|
-
missing_fields: list[str]
|
|
54
|
-
extra_fields: list[str]
|
|
55
|
-
|
|
56
|
-
def with_default_values(
|
|
57
|
-
self, values: Mapping[str, MigrationValue]
|
|
58
|
-
) -> "MigrationCandidate":
|
|
59
|
-
if not values:
|
|
60
|
-
return self
|
|
61
|
-
updated_defaults = dict(self.defaults_applied)
|
|
62
|
-
updated_defaults.update(values)
|
|
63
|
-
return _rebuild_candidate_with_defaults(self, dict(values), updated_defaults)
|
|
64
|
-
|
|
65
|
-
|
|
66
44
|
@dataclass(frozen=True)
|
|
67
45
|
class MigrationSkip:
|
|
68
|
-
|
|
46
|
+
source: FuruRef
|
|
69
47
|
reason: str
|
|
70
48
|
|
|
71
49
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
) -> MigrationCandidate:
|
|
77
|
-
target_class = _resolve_target_class(candidate.to_namespace)
|
|
78
|
-
updated_config = _typed_config(dict(candidate.to_config))
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class MigrationReport:
|
|
52
|
+
records: list[MigrationRecord]
|
|
53
|
+
skips: list[MigrationSkip]
|
|
79
54
|
|
|
80
|
-
target_fields = _target_field_names(target_class)
|
|
81
|
-
config_keys = set(updated_config.keys()) - {"__class__"}
|
|
82
|
-
conflicts = set(new_defaults) & config_keys
|
|
83
|
-
if conflicts:
|
|
84
|
-
raise ValueError(
|
|
85
|
-
"migration: default_values provided for existing fields: "
|
|
86
|
-
f"{_format_fields(conflicts)}"
|
|
87
|
-
)
|
|
88
|
-
unknown = set(new_defaults) - set(target_fields)
|
|
89
|
-
if unknown:
|
|
90
|
-
raise ValueError(
|
|
91
|
-
"migration: default_values contains fields not in target schema: "
|
|
92
|
-
f"{_format_fields(unknown)}"
|
|
93
|
-
)
|
|
94
55
|
|
|
95
|
-
|
|
96
|
-
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class _Source:
|
|
58
|
+
ref: FuruRef
|
|
59
|
+
furu_obj: dict[str, JsonValue]
|
|
60
|
+
migration: MigrationRecord | None
|
|
97
61
|
|
|
98
|
-
updated_config["__class__"] = candidate.to_namespace
|
|
99
|
-
_typecheck_config(updated_config)
|
|
100
62
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
63
|
+
def migrate(
|
|
64
|
+
cls: _FuruClass,
|
|
65
|
+
*,
|
|
66
|
+
from_schema: Iterable[str] | None = None,
|
|
67
|
+
from_drop: Iterable[str] | None = None,
|
|
68
|
+
from_add: Iterable[str] | None = None,
|
|
69
|
+
from_hash: str | None = None,
|
|
70
|
+
from_furu_obj: dict[str, JsonValue] | None = None,
|
|
71
|
+
from_namespace: str | None = None,
|
|
72
|
+
default_field: Iterable[str] | None = None,
|
|
73
|
+
set_field: Mapping[str, JsonValue] | None = None,
|
|
74
|
+
drop_field: Iterable[str] | None = None,
|
|
75
|
+
rename_field: Mapping[str, str] | None = None,
|
|
76
|
+
include_alias_sources: bool = False,
|
|
77
|
+
conflict: MigrationConflict = "throw",
|
|
78
|
+
origin: str | None = None,
|
|
79
|
+
note: str | None = None,
|
|
80
|
+
) -> MigrationReport:
|
|
81
|
+
"""
|
|
82
|
+
Migrate stored objects to the current schema via alias-only records.
|
|
83
|
+
|
|
84
|
+
Notes:
|
|
85
|
+
set_field only adds fields that are not already present. To replace a
|
|
86
|
+
field, drop it first and then set it.
|
|
87
|
+
"""
|
|
88
|
+
selector_count = sum(
|
|
89
|
+
1
|
|
90
|
+
for value in (
|
|
91
|
+
from_schema is not None,
|
|
92
|
+
from_hash is not None,
|
|
93
|
+
from_furu_obj is not None,
|
|
94
|
+
from_drop is not None or from_add is not None,
|
|
107
95
|
)
|
|
108
|
-
|
|
109
|
-
|
|
96
|
+
if value
|
|
97
|
+
)
|
|
98
|
+
if selector_count == 0:
|
|
110
99
|
raise ValueError(
|
|
111
|
-
"migration:
|
|
112
|
-
f"{_format_fields(extra_fields)}"
|
|
100
|
+
"migration: provide one of from_schema, from_hash, from_furu_obj, or from_drop/from_add"
|
|
113
101
|
)
|
|
102
|
+
if selector_count > 1:
|
|
103
|
+
raise ValueError("migration: source selection is ambiguous")
|
|
114
104
|
|
|
115
|
-
|
|
116
|
-
to_ref = _build_target_ref(target_class, candidate.to_namespace, to_hash)
|
|
117
|
-
return MigrationCandidate(
|
|
118
|
-
from_ref=candidate.from_ref,
|
|
119
|
-
to_ref=to_ref,
|
|
120
|
-
to_namespace=candidate.to_namespace,
|
|
121
|
-
to_config=updated_config,
|
|
122
|
-
defaults_applied=defaults_applied,
|
|
123
|
-
fields_dropped=candidate.fields_dropped,
|
|
124
|
-
missing_fields=missing_fields,
|
|
125
|
-
extra_fields=extra_fields,
|
|
126
|
-
)
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
MigrationPolicy = Literal["alias", "move", "copy"]
|
|
130
|
-
MigrationConflict = Literal["throw", "skip", "overwrite"]
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
@overload
|
|
134
|
-
def find_migration_candidates(
|
|
135
|
-
*,
|
|
136
|
-
namespace: str,
|
|
137
|
-
to_obj: type[Furu],
|
|
138
|
-
default_values: Mapping[str, MigrationValue] | None = None,
|
|
139
|
-
default_fields: Iterable[str] | None = None,
|
|
140
|
-
drop_fields: Iterable[str] | None = None,
|
|
141
|
-
) -> list[MigrationCandidate]: ...
|
|
142
|
-
|
|
105
|
+
namespace = from_namespace or _namespace_str(cls)
|
|
143
106
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
namespace: str | NamespacePair,
|
|
158
|
-
to_obj: type[Furu] | None = None,
|
|
159
|
-
default_values: Mapping[str, MigrationValue] | None = None,
|
|
160
|
-
default_fields: Iterable[str] | None = None,
|
|
161
|
-
drop_fields: Iterable[str] | None = None,
|
|
162
|
-
) -> list[MigrationCandidate]:
|
|
163
|
-
if namespace is None:
|
|
164
|
-
raise ValueError("migration: namespace is required")
|
|
165
|
-
if isinstance(namespace, NamespacePair):
|
|
166
|
-
if to_obj is not None:
|
|
167
|
-
raise ValueError("migration: to_obj cannot be used with NamespacePair")
|
|
168
|
-
from_namespace = namespace.from_namespace
|
|
169
|
-
to_namespace = namespace.to_namespace
|
|
170
|
-
target_class = _resolve_target_class(to_namespace)
|
|
171
|
-
elif isinstance(namespace, str):
|
|
172
|
-
if not _is_furu_class(to_obj):
|
|
173
|
-
raise ValueError(
|
|
174
|
-
"migration: to_obj must be a class (use find_migration_candidates_initialized_target for instances)"
|
|
107
|
+
if from_schema is not None:
|
|
108
|
+
from_schema_key = _normalize_schema(from_schema)
|
|
109
|
+
sources = _sources_from_schema(
|
|
110
|
+
namespace,
|
|
111
|
+
from_schema_key,
|
|
112
|
+
include_alias_sources=include_alias_sources,
|
|
113
|
+
)
|
|
114
|
+
elif from_hash is not None:
|
|
115
|
+
sources = [
|
|
116
|
+
_source_from_hash(
|
|
117
|
+
namespace,
|
|
118
|
+
from_hash,
|
|
119
|
+
include_alias_sources=include_alias_sources,
|
|
175
120
|
)
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
121
|
+
]
|
|
122
|
+
elif from_furu_obj is not None:
|
|
123
|
+
sources = [
|
|
124
|
+
_source_from_furu_obj(
|
|
125
|
+
namespace,
|
|
126
|
+
from_furu_obj,
|
|
127
|
+
include_alias_sources=include_alias_sources,
|
|
181
128
|
)
|
|
182
|
-
|
|
129
|
+
]
|
|
183
130
|
else:
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
from_ref,
|
|
190
|
-
config,
|
|
191
|
-
to_namespace=to_namespace,
|
|
192
|
-
target_class=target_class,
|
|
193
|
-
default_values=default_values,
|
|
194
|
-
default_fields=default_fields,
|
|
195
|
-
drop_fields=drop_fields,
|
|
196
|
-
default_source=None,
|
|
131
|
+
from_schema_key = _schema_from_drop_add(cls, from_drop, from_add)
|
|
132
|
+
sources = _sources_from_schema(
|
|
133
|
+
namespace,
|
|
134
|
+
from_schema_key,
|
|
135
|
+
include_alias_sources=include_alias_sources,
|
|
197
136
|
)
|
|
198
|
-
candidates.append(candidate)
|
|
199
|
-
return candidates
|
|
200
137
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
)
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
138
|
+
if not sources:
|
|
139
|
+
return MigrationReport(records=[], skips=[])
|
|
140
|
+
|
|
141
|
+
alias_index = collect_aliases(include_inactive=True)
|
|
142
|
+
alias_schema_cache: dict[Path, tuple[str, ...]] = {}
|
|
143
|
+
seen_alias_schema: set[tuple[tuple[str, str, RootKind], tuple[str, ...]]] = set()
|
|
144
|
+
|
|
145
|
+
rename_map = dict(rename_field) if rename_field is not None else {}
|
|
146
|
+
drop_list = list(drop_field) if drop_field is not None else []
|
|
147
|
+
default_list = list(default_field) if default_field is not None else []
|
|
148
|
+
set_map = dict(set_field) if set_field is not None else {}
|
|
149
|
+
|
|
150
|
+
records: list[MigrationRecord] = []
|
|
151
|
+
skips: list[MigrationSkip] = []
|
|
152
|
+
|
|
153
|
+
for source in sources:
|
|
154
|
+
original_ref = resolve_original_ref(source.ref)
|
|
155
|
+
_ensure_original_success(original_ref)
|
|
156
|
+
|
|
157
|
+
target_fields = schema_key_from_cls(cast(type, cls))
|
|
158
|
+
updated_fields = _apply_transforms(
|
|
159
|
+
source.furu_obj,
|
|
160
|
+
target_fields=target_fields,
|
|
161
|
+
rename_field=rename_map,
|
|
162
|
+
drop_field=drop_list,
|
|
163
|
+
default_field=default_list,
|
|
164
|
+
set_field=set_map,
|
|
165
|
+
target_class=cls,
|
|
216
166
|
)
|
|
217
167
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
target_config = FuruSerializer.to_dict(to_obj)
|
|
223
|
-
if not isinstance(target_config, dict):
|
|
224
|
-
raise TypeError("migration: to_obj must serialize to a dict")
|
|
225
|
-
target_config = _typed_config(target_config)
|
|
226
|
-
target_config["__class__"] = to_namespace
|
|
227
|
-
_typecheck_config(target_config)
|
|
228
|
-
|
|
229
|
-
candidates: list[MigrationCandidate] = []
|
|
230
|
-
for from_ref, config in _iter_source_configs(source_namespace):
|
|
231
|
-
candidate = _build_candidate(
|
|
232
|
-
from_ref,
|
|
233
|
-
config,
|
|
234
|
-
to_namespace=to_namespace,
|
|
235
|
-
target_class=target_class,
|
|
236
|
-
default_values=None,
|
|
237
|
-
default_fields=default_fields,
|
|
238
|
-
drop_fields=drop_fields,
|
|
239
|
-
default_source=to_obj,
|
|
240
|
-
)
|
|
241
|
-
aligned = _align_candidate_to_target(candidate, target_config)
|
|
242
|
-
if aligned is not None:
|
|
243
|
-
candidates.append(aligned)
|
|
244
|
-
return candidates
|
|
168
|
+
target_namespace = _namespace_str(cls)
|
|
169
|
+
target_obj = dict(updated_fields)
|
|
170
|
+
target_obj["__class__"] = target_namespace
|
|
245
171
|
|
|
172
|
+
obj = FuruSerializer.from_dict(target_obj)
|
|
173
|
+
target_hash = FuruSerializer.compute_hash(obj)
|
|
174
|
+
target_ref = _target_ref(cls, target_hash)
|
|
175
|
+
target_schema_key = schema_key_from_furu_obj(FuruSerializer.to_dict(obj))
|
|
246
176
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
note: str | None = None,
|
|
255
|
-
conflict: Literal["throw", "overwrite"] = "throw",
|
|
256
|
-
) -> list[MigrationRecord]: ...
|
|
177
|
+
alias_key = (original_ref.namespace, original_ref.furu_hash, original_ref.root)
|
|
178
|
+
if (alias_key, target_schema_key) in seen_alias_schema:
|
|
179
|
+
reason = "migration: alias schema already created in this run"
|
|
180
|
+
if conflict == "skip":
|
|
181
|
+
skips.append(MigrationSkip(source=source.ref, reason=reason))
|
|
182
|
+
continue
|
|
183
|
+
raise ValueError(reason)
|
|
257
184
|
|
|
185
|
+
if _alias_schema_conflict(
|
|
186
|
+
alias_index,
|
|
187
|
+
alias_schema_cache,
|
|
188
|
+
alias_key,
|
|
189
|
+
target_schema_key,
|
|
190
|
+
):
|
|
191
|
+
reason = "migration: alias schema already exists for original"
|
|
192
|
+
if conflict == "skip":
|
|
193
|
+
skips.append(MigrationSkip(source=source.ref, reason=reason))
|
|
194
|
+
continue
|
|
195
|
+
raise ValueError(reason)
|
|
196
|
+
|
|
197
|
+
record = _write_alias(
|
|
198
|
+
target_obj=obj,
|
|
199
|
+
original_ref=original_ref,
|
|
200
|
+
target_ref=target_ref,
|
|
201
|
+
source_ref=source.ref,
|
|
202
|
+
skips=skips,
|
|
203
|
+
conflict=conflict,
|
|
204
|
+
origin=origin,
|
|
205
|
+
note=note,
|
|
206
|
+
)
|
|
207
|
+
if record is None:
|
|
208
|
+
continue
|
|
209
|
+
records.append(record)
|
|
210
|
+
seen_alias_schema.add((alias_key, target_schema_key))
|
|
258
211
|
|
|
259
|
-
|
|
260
|
-
def apply_migration(
|
|
261
|
-
candidate: MigrationCandidate,
|
|
262
|
-
*,
|
|
263
|
-
policy: MigrationPolicy = "alias",
|
|
264
|
-
cascade: bool = True,
|
|
265
|
-
origin: str | None = None,
|
|
266
|
-
note: str | None = None,
|
|
267
|
-
conflict: Literal["skip"],
|
|
268
|
-
) -> list[MigrationRecord | MigrationSkip]: ...
|
|
212
|
+
return MigrationReport(records=records, skips=skips)
|
|
269
213
|
|
|
270
214
|
|
|
271
|
-
def
|
|
272
|
-
|
|
215
|
+
def migrate_one(
|
|
216
|
+
cls: _FuruClass,
|
|
273
217
|
*,
|
|
274
|
-
|
|
275
|
-
|
|
218
|
+
from_hash: str,
|
|
219
|
+
from_namespace: str | None = None,
|
|
220
|
+
default_field: Iterable[str] | None = None,
|
|
221
|
+
set_field: Mapping[str, JsonValue] | None = None,
|
|
222
|
+
drop_field: Iterable[str] | None = None,
|
|
223
|
+
rename_field: Mapping[str, str] | None = None,
|
|
224
|
+
include_alias_sources: bool = False,
|
|
225
|
+
conflict: MigrationConflict = "throw",
|
|
276
226
|
origin: str | None = None,
|
|
277
227
|
note: str | None = None,
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
else [_CascadeNode(candidate=candidate, parent=None)]
|
|
228
|
+
) -> MigrationRecord | None:
|
|
229
|
+
report = migrate(
|
|
230
|
+
cls,
|
|
231
|
+
from_hash=from_hash,
|
|
232
|
+
from_namespace=from_namespace,
|
|
233
|
+
default_field=default_field,
|
|
234
|
+
set_field=set_field,
|
|
235
|
+
drop_field=drop_field,
|
|
236
|
+
rename_field=rename_field,
|
|
237
|
+
include_alias_sources=include_alias_sources,
|
|
238
|
+
conflict=conflict,
|
|
239
|
+
origin=origin,
|
|
240
|
+
note=note,
|
|
292
241
|
)
|
|
293
|
-
|
|
242
|
+
if report.records:
|
|
243
|
+
return report.records[0]
|
|
244
|
+
return None
|
|
294
245
|
|
|
295
|
-
conflict_statuses: dict[_CandidateKey, str] = {}
|
|
296
|
-
for node in cascade_nodes:
|
|
297
|
-
status = _target_status(node.candidate)
|
|
298
|
-
if status is not None:
|
|
299
|
-
conflict_statuses[node.key] = status
|
|
300
246
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
247
|
+
def current(
|
|
248
|
+
cls: _FuruClass,
|
|
249
|
+
*,
|
|
250
|
+
namespace: str | None = None,
|
|
251
|
+
) -> list[FuruRef]:
|
|
252
|
+
target_schema = schema_key_from_cls(cast(type, cls))
|
|
253
|
+
return _refs_by_schema(namespace or _namespace_str(cls), target_schema, match=True)
|
|
306
254
|
|
|
307
|
-
skip_keys: set[_CandidateKey] = set()
|
|
308
|
-
if conflict == "skip" and conflict_statuses:
|
|
309
|
-
skip_keys = _expand_skip_keys(conflict_statuses.keys(), parent_map)
|
|
310
|
-
for key in conflict_statuses:
|
|
311
|
-
status = conflict_statuses[key]
|
|
312
|
-
get_logger().warning(
|
|
313
|
-
"migration: skipping candidate due to target status %s",
|
|
314
|
-
status,
|
|
315
|
-
)
|
|
316
255
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
256
|
+
def stale(
|
|
257
|
+
cls: _FuruClass,
|
|
258
|
+
*,
|
|
259
|
+
namespace: str | None = None,
|
|
260
|
+
) -> list[FuruRef]:
|
|
261
|
+
target_schema = schema_key_from_cls(cast(type, cls))
|
|
262
|
+
return _refs_by_schema(namespace or _namespace_str(cls), target_schema, match=False)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def resolve_original_ref(ref: FuruRef) -> FuruRef:
|
|
266
|
+
current = ref
|
|
267
|
+
seen: set[tuple[str, str, RootKind]] = set()
|
|
268
|
+
while True:
|
|
269
|
+
record = MigrationManager.read_migration(current.directory)
|
|
270
|
+
if record is None or record.kind != "alias":
|
|
271
|
+
return current
|
|
272
|
+
key = (record.from_namespace, record.from_hash, record.from_root)
|
|
273
|
+
if key in seen:
|
|
274
|
+
raise ValueError("migration: alias loop detected")
|
|
275
|
+
seen.add(key)
|
|
276
|
+
directory = MigrationManager.resolve_dir(record, target="from")
|
|
277
|
+
current = FuruRef(
|
|
278
|
+
namespace=record.from_namespace,
|
|
279
|
+
furu_hash=record.from_hash,
|
|
280
|
+
root=record.from_root,
|
|
281
|
+
directory=directory,
|
|
333
282
|
)
|
|
334
|
-
results.append(record)
|
|
335
|
-
return results
|
|
336
|
-
|
|
337
283
|
|
|
338
|
-
@dataclass(frozen=True)
|
|
339
|
-
class _CascadeNode:
|
|
340
|
-
candidate: MigrationCandidate
|
|
341
|
-
parent: _CandidateKey | None
|
|
342
284
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
285
|
+
def _schema_from_drop_add(
|
|
286
|
+
cls: _FuruClass,
|
|
287
|
+
from_drop: Iterable[str] | None,
|
|
288
|
+
from_add: Iterable[str] | None,
|
|
289
|
+
) -> tuple[str, ...]:
|
|
290
|
+
current = set(schema_key_from_cls(cast(type, cls)))
|
|
346
291
|
|
|
292
|
+
if from_drop is not None:
|
|
293
|
+
drop_fields = _normalize_schema(from_drop)
|
|
294
|
+
missing = set(drop_fields) - current
|
|
295
|
+
if missing:
|
|
296
|
+
raise ValueError(
|
|
297
|
+
f"migration: from_drop fields not in current schema: {_format_fields(missing)}"
|
|
298
|
+
)
|
|
299
|
+
current -= set(drop_fields)
|
|
347
300
|
|
|
348
|
-
|
|
301
|
+
if from_add is not None:
|
|
302
|
+
add_fields = _normalize_schema(from_add)
|
|
303
|
+
overlap = set(add_fields) & current
|
|
304
|
+
if overlap:
|
|
305
|
+
raise ValueError(
|
|
306
|
+
f"migration: from_add fields already in current schema: {_format_fields(overlap)}"
|
|
307
|
+
)
|
|
308
|
+
current |= set(add_fields)
|
|
349
309
|
|
|
310
|
+
return tuple(sorted(current))
|
|
350
311
|
|
|
351
|
-
def _candidate_key(candidate: MigrationCandidate) -> _CandidateKey:
|
|
352
|
-
return (
|
|
353
|
-
candidate.from_ref.namespace,
|
|
354
|
-
candidate.from_ref.furu_hash,
|
|
355
|
-
candidate.from_ref.root,
|
|
356
|
-
)
|
|
357
312
|
|
|
313
|
+
def _normalize_schema(values: Iterable[str]) -> tuple[str, ...]:
|
|
314
|
+
keys: set[str] = set()
|
|
315
|
+
for value in values:
|
|
316
|
+
if not isinstance(value, str):
|
|
317
|
+
raise TypeError("migration: schema fields must be strings")
|
|
318
|
+
if value.startswith("_"):
|
|
319
|
+
raise ValueError("migration: schema fields cannot start with '_'")
|
|
320
|
+
keys.add(value)
|
|
321
|
+
return tuple(sorted(keys))
|
|
358
322
|
|
|
359
|
-
def _build_cascade_candidates(root: MigrationCandidate) -> list[_CascadeNode]:
|
|
360
|
-
nodes: list[_CascadeNode] = []
|
|
361
|
-
queue: list[_CascadeNode] = [_CascadeNode(candidate=root, parent=None)]
|
|
362
|
-
seen: set[_CandidateKey] = {_candidate_key(root)}
|
|
363
323
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
324
|
+
def _sources_from_schema(
|
|
325
|
+
namespace: str,
|
|
326
|
+
schema_key: tuple[str, ...],
|
|
327
|
+
*,
|
|
328
|
+
include_alias_sources: bool,
|
|
329
|
+
) -> list[_Source]:
|
|
330
|
+
sources: list[_Source] = []
|
|
331
|
+
for ref, metadata in _iter_namespace_metadata(
|
|
332
|
+
namespace,
|
|
333
|
+
include_alias_sources=include_alias_sources,
|
|
334
|
+
):
|
|
335
|
+
if schema_key_from_metadata_raw(metadata) != schema_key:
|
|
336
|
+
continue
|
|
337
|
+
furu_obj = metadata.get("furu_obj")
|
|
338
|
+
if not isinstance(furu_obj, dict):
|
|
339
|
+
raise TypeError("migration: metadata furu_obj must be a dict")
|
|
340
|
+
migration = MigrationManager.read_migration(ref.directory)
|
|
341
|
+
sources.append(_Source(ref=ref, furu_obj=furu_obj, migration=migration))
|
|
342
|
+
return sources
|
|
374
343
|
|
|
375
344
|
|
|
376
|
-
def
|
|
377
|
-
|
|
345
|
+
def _source_from_hash(
|
|
346
|
+
namespace: str,
|
|
347
|
+
furu_hash: str,
|
|
348
|
+
*,
|
|
349
|
+
include_alias_sources: bool,
|
|
350
|
+
) -> _Source:
|
|
351
|
+
ref = _find_ref_by_hash(namespace, furu_hash)
|
|
352
|
+
metadata = MetadataManager.read_metadata_raw(ref.directory)
|
|
378
353
|
if metadata is None:
|
|
379
|
-
|
|
354
|
+
raise FileNotFoundError(f"migration: metadata not found for {ref.directory}")
|
|
380
355
|
furu_obj = metadata.get("furu_obj")
|
|
381
356
|
if not isinstance(furu_obj, dict):
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
)
|
|
391
|
-
if not changed:
|
|
392
|
-
continue
|
|
393
|
-
dependent_namespace = _extract_namespace(updated_config)
|
|
394
|
-
target_class = _resolve_target_class(dependent_namespace)
|
|
395
|
-
dependent_candidate = _build_candidate(
|
|
396
|
-
ref,
|
|
397
|
-
updated_config,
|
|
398
|
-
to_namespace=dependent_namespace,
|
|
399
|
-
target_class=target_class,
|
|
400
|
-
default_values=None,
|
|
401
|
-
default_fields=None,
|
|
402
|
-
drop_fields=None,
|
|
403
|
-
default_source=None,
|
|
357
|
+
raise TypeError("migration: metadata furu_obj must be a dict")
|
|
358
|
+
migration = MigrationManager.read_migration(ref.directory)
|
|
359
|
+
if not _alias_source_allowed(
|
|
360
|
+
migration,
|
|
361
|
+
include_alias_sources=include_alias_sources,
|
|
362
|
+
):
|
|
363
|
+
raise ValueError(
|
|
364
|
+
"migration: source is an alias; set include_alias_sources=True"
|
|
404
365
|
)
|
|
405
|
-
|
|
406
|
-
dependent_candidate.to_ref.furu_hash
|
|
407
|
-
== dependent_candidate.from_ref.furu_hash
|
|
408
|
-
):
|
|
409
|
-
continue
|
|
410
|
-
dependents.append(dependent_candidate)
|
|
411
|
-
return dependents
|
|
366
|
+
return _Source(ref=ref, furu_obj=furu_obj, migration=migration)
|
|
412
367
|
|
|
413
368
|
|
|
414
|
-
def
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
to_config: dict[str, JsonValue],
|
|
418
|
-
) -> tuple[JsonValue, bool]:
|
|
419
|
-
if isinstance(value, dict):
|
|
420
|
-
if "__class__" in value:
|
|
421
|
-
if FuruSerializer.compute_hash(value) == from_hash:
|
|
422
|
-
return dict(to_config), True
|
|
423
|
-
changed = False
|
|
424
|
-
updated: dict[str, JsonValue] = {}
|
|
425
|
-
for key, item in value.items():
|
|
426
|
-
new_value, was_changed = _replace_dependency(item, from_hash, to_config)
|
|
427
|
-
if was_changed:
|
|
428
|
-
changed = True
|
|
429
|
-
updated[key] = new_value
|
|
430
|
-
return updated, changed
|
|
431
|
-
if isinstance(value, list):
|
|
432
|
-
updated_list: list[JsonValue] = []
|
|
433
|
-
changed = False
|
|
434
|
-
for item in value:
|
|
435
|
-
new_value, was_changed = _replace_dependency(item, from_hash, to_config)
|
|
436
|
-
if was_changed:
|
|
437
|
-
changed = True
|
|
438
|
-
updated_list.append(new_value)
|
|
439
|
-
return updated_list, changed
|
|
440
|
-
return value, False
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
def _apply_single_migration(
|
|
444
|
-
candidate: MigrationCandidate,
|
|
369
|
+
def _source_from_furu_obj(
|
|
370
|
+
namespace: str,
|
|
371
|
+
furu_obj: dict[str, JsonValue],
|
|
445
372
|
*,
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
to_dir = candidate.to_ref.directory
|
|
454
|
-
|
|
455
|
-
if conflict == "overwrite" and to_dir.exists():
|
|
456
|
-
shutil.rmtree(to_dir)
|
|
457
|
-
|
|
458
|
-
to_dir.mkdir(parents=True, exist_ok=True)
|
|
459
|
-
StateManager.ensure_internal_dir(to_dir)
|
|
460
|
-
now = _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
|
|
461
|
-
|
|
462
|
-
if policy in {"move", "copy"}:
|
|
463
|
-
_transfer_payload(from_dir, to_dir, policy)
|
|
464
|
-
_copy_state(from_dir, to_dir, clear_source=policy == "move")
|
|
465
|
-
else:
|
|
466
|
-
_write_migrated_state(to_dir)
|
|
467
|
-
|
|
468
|
-
to_obj = FuruSerializer.from_dict(candidate.to_config)
|
|
469
|
-
metadata = MetadataManager.create_metadata(to_obj, to_dir, ignore_diff=True)
|
|
470
|
-
MetadataManager.write_metadata(metadata, to_dir)
|
|
471
|
-
|
|
472
|
-
default_values = _serialize_default_values(candidate.defaults_applied)
|
|
473
|
-
record = MigrationRecord(
|
|
474
|
-
kind=_kind_for_policy(policy),
|
|
475
|
-
policy=policy,
|
|
476
|
-
from_namespace=candidate.from_ref.namespace,
|
|
477
|
-
from_hash=candidate.from_ref.furu_hash,
|
|
478
|
-
from_root=candidate.from_ref.root,
|
|
479
|
-
to_namespace=candidate.to_ref.namespace,
|
|
480
|
-
to_hash=candidate.to_ref.furu_hash,
|
|
481
|
-
to_root=candidate.to_ref.root,
|
|
482
|
-
migrated_at=now,
|
|
483
|
-
overwritten_at=None,
|
|
484
|
-
default_values=default_values,
|
|
485
|
-
origin=origin,
|
|
486
|
-
note=note,
|
|
373
|
+
include_alias_sources: bool,
|
|
374
|
+
) -> _Source:
|
|
375
|
+
furu_hash = FuruSerializer.compute_hash(furu_obj)
|
|
376
|
+
return _source_from_hash(
|
|
377
|
+
namespace,
|
|
378
|
+
furu_hash,
|
|
379
|
+
include_alias_sources=include_alias_sources,
|
|
487
380
|
)
|
|
488
|
-
MigrationManager.write_migration(record, to_dir)
|
|
489
|
-
|
|
490
|
-
if policy != "copy":
|
|
491
|
-
from_record = MigrationRecord(
|
|
492
|
-
kind="migrated",
|
|
493
|
-
policy=policy,
|
|
494
|
-
from_namespace=candidate.from_ref.namespace,
|
|
495
|
-
from_hash=candidate.from_ref.furu_hash,
|
|
496
|
-
from_root=candidate.from_ref.root,
|
|
497
|
-
to_namespace=candidate.to_ref.namespace,
|
|
498
|
-
to_hash=candidate.to_ref.furu_hash,
|
|
499
|
-
to_root=candidate.to_ref.root,
|
|
500
|
-
migrated_at=now,
|
|
501
|
-
overwritten_at=None,
|
|
502
|
-
default_values=default_values,
|
|
503
|
-
origin=origin,
|
|
504
|
-
note=note,
|
|
505
|
-
)
|
|
506
|
-
MigrationManager.write_migration(from_record, from_dir)
|
|
507
|
-
|
|
508
|
-
event: dict[str, str | int] = {
|
|
509
|
-
"type": "migrated",
|
|
510
|
-
"policy": policy,
|
|
511
|
-
"from_namespace": candidate.from_ref.namespace,
|
|
512
|
-
"from_hash": candidate.from_ref.furu_hash,
|
|
513
|
-
"to_namespace": candidate.to_ref.namespace,
|
|
514
|
-
"to_hash": candidate.to_ref.furu_hash,
|
|
515
|
-
}
|
|
516
|
-
if default_values is not None:
|
|
517
|
-
event["default_values"] = json.dumps(default_values, sort_keys=True)
|
|
518
|
-
StateManager.append_event(to_dir, event)
|
|
519
|
-
StateManager.append_event(from_dir, event.copy())
|
|
520
|
-
|
|
521
|
-
if conflict == "overwrite" and conflict_status is not None:
|
|
522
|
-
overwrite_event = {
|
|
523
|
-
"type": "migration_overwrite",
|
|
524
|
-
"policy": policy,
|
|
525
|
-
"from_namespace": candidate.from_ref.namespace,
|
|
526
|
-
"from_hash": candidate.from_ref.furu_hash,
|
|
527
|
-
"to_namespace": candidate.to_ref.namespace,
|
|
528
|
-
"to_hash": candidate.to_ref.furu_hash,
|
|
529
|
-
"reason": "force_overwrite",
|
|
530
|
-
}
|
|
531
|
-
StateManager.append_event(to_dir, overwrite_event)
|
|
532
|
-
StateManager.append_event(from_dir, overwrite_event.copy())
|
|
533
|
-
|
|
534
|
-
get_logger().info(
|
|
535
|
-
"migration: %s -> %s (%s)",
|
|
536
|
-
from_dir,
|
|
537
|
-
to_dir,
|
|
538
|
-
policy,
|
|
539
|
-
)
|
|
540
|
-
return record
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
def _transfer_payload(from_dir: Path, to_dir: Path, policy: MigrationPolicy) -> None:
|
|
544
|
-
for item in from_dir.iterdir():
|
|
545
|
-
if item.name == StateManager.INTERNAL_DIR:
|
|
546
|
-
continue
|
|
547
|
-
destination = to_dir / item.name
|
|
548
|
-
if policy == "move":
|
|
549
|
-
shutil.move(str(item), destination)
|
|
550
|
-
continue
|
|
551
|
-
if item.is_dir():
|
|
552
|
-
shutil.copytree(item, destination, dirs_exist_ok=True)
|
|
553
|
-
else:
|
|
554
|
-
shutil.copy2(item, destination)
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
def _copy_state(from_dir: Path, to_dir: Path, *, clear_source: bool) -> None:
|
|
558
|
-
src_internal = from_dir / StateManager.INTERNAL_DIR
|
|
559
|
-
if not src_internal.exists():
|
|
560
|
-
return
|
|
561
|
-
state_path = StateManager.get_state_path(from_dir)
|
|
562
|
-
if state_path.is_file():
|
|
563
|
-
shutil.copy2(state_path, StateManager.get_state_path(to_dir))
|
|
564
|
-
success_marker = StateManager.get_success_marker_path(from_dir)
|
|
565
|
-
if success_marker.is_file():
|
|
566
|
-
shutil.copy2(success_marker, StateManager.get_success_marker_path(to_dir))
|
|
567
|
-
if clear_source:
|
|
568
|
-
_write_migrated_state(from_dir)
|
|
569
|
-
StateManager.get_success_marker_path(from_dir).unlink(missing_ok=True)
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
def _write_migrated_state(directory: Path) -> None:
|
|
573
|
-
def mutate(state) -> None:
|
|
574
|
-
state.result = _StateResultMigrated(status="migrated")
|
|
575
|
-
state.attempt = None
|
|
576
|
-
|
|
577
|
-
StateManager.update_state(directory, mutate)
|
|
578
381
|
|
|
579
382
|
|
|
580
|
-
def
|
|
581
|
-
if policy == "alias":
|
|
582
|
-
return "alias"
|
|
583
|
-
if policy == "move":
|
|
584
|
-
return "moved"
|
|
585
|
-
if policy == "copy":
|
|
586
|
-
return "copied"
|
|
587
|
-
raise ValueError(f"Unsupported migration policy: {policy}")
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
def _iter_source_configs(
|
|
383
|
+
def _iter_namespace_metadata(
|
|
591
384
|
namespace: str,
|
|
385
|
+
*,
|
|
386
|
+
include_alias_sources: bool,
|
|
592
387
|
) -> Iterable[tuple[FuruRef, dict[str, JsonValue]]]:
|
|
593
388
|
namespace_path = Path(*namespace.split("."))
|
|
594
389
|
for version_controlled in (False, True):
|
|
@@ -602,254 +397,254 @@ def _iter_source_configs(
|
|
|
602
397
|
metadata = MetadataManager.read_metadata_raw(entry)
|
|
603
398
|
if metadata is None:
|
|
604
399
|
continue
|
|
605
|
-
|
|
606
|
-
if not
|
|
400
|
+
migration = MigrationManager.read_migration(entry)
|
|
401
|
+
if not _alias_source_allowed(
|
|
402
|
+
migration,
|
|
403
|
+
include_alias_sources=include_alias_sources,
|
|
404
|
+
):
|
|
607
405
|
continue
|
|
608
|
-
root_kind:
|
|
406
|
+
root_kind: RootKind = "git" if version_controlled else "data"
|
|
609
407
|
ref = FuruRef(
|
|
610
408
|
namespace=namespace,
|
|
611
409
|
furu_hash=entry.name,
|
|
612
410
|
root=root_kind,
|
|
613
411
|
directory=entry,
|
|
614
412
|
)
|
|
615
|
-
yield ref,
|
|
413
|
+
yield ref, metadata
|
|
616
414
|
|
|
617
415
|
|
|
618
|
-
def
|
|
416
|
+
def _alias_source_allowed(
|
|
417
|
+
migration: MigrationRecord | None,
|
|
418
|
+
*,
|
|
419
|
+
include_alias_sources: bool,
|
|
420
|
+
) -> bool:
|
|
421
|
+
if migration is None:
|
|
422
|
+
return True
|
|
423
|
+
if migration.kind != "alias":
|
|
424
|
+
return True
|
|
425
|
+
return include_alias_sources
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _find_ref_by_hash(namespace: str, furu_hash: str) -> FuruRef:
|
|
429
|
+
namespace_path = Path(*namespace.split("."))
|
|
430
|
+
matches: list[FuruRef] = []
|
|
619
431
|
for version_controlled in (False, True):
|
|
620
432
|
root = FURU_CONFIG.get_root(version_controlled=version_controlled)
|
|
621
|
-
|
|
433
|
+
directory = root / namespace_path / furu_hash
|
|
434
|
+
if not directory.is_dir():
|
|
622
435
|
continue
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
state_path = StateManager.get_state_path(namespace_dir)
|
|
627
|
-
if not state_path.is_file():
|
|
628
|
-
continue
|
|
629
|
-
metadata = MetadataManager.read_metadata_raw(namespace_dir)
|
|
630
|
-
if metadata is None:
|
|
631
|
-
continue
|
|
632
|
-
furu_obj = metadata.get("furu_obj")
|
|
633
|
-
if not isinstance(furu_obj, dict):
|
|
634
|
-
continue
|
|
635
|
-
namespace = ".".join(namespace_dir.relative_to(root).parts[:-1])
|
|
636
|
-
root_kind: str = "git" if version_controlled else "data"
|
|
637
|
-
ref = FuruRef(
|
|
436
|
+
root_kind: RootKind = "git" if version_controlled else "data"
|
|
437
|
+
matches.append(
|
|
438
|
+
FuruRef(
|
|
638
439
|
namespace=namespace,
|
|
639
|
-
furu_hash=
|
|
440
|
+
furu_hash=furu_hash,
|
|
640
441
|
root=root_kind,
|
|
641
|
-
directory=
|
|
442
|
+
directory=directory,
|
|
642
443
|
)
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
*,
|
|
650
|
-
to_namespace: str,
|
|
651
|
-
target_class: type[Furu],
|
|
652
|
-
default_values: Mapping[str, MigrationValue] | None,
|
|
653
|
-
default_fields: Iterable[str] | None,
|
|
654
|
-
drop_fields: Iterable[str] | None,
|
|
655
|
-
default_source: Furu | None,
|
|
656
|
-
) -> MigrationCandidate:
|
|
657
|
-
config = dict(source_config)
|
|
658
|
-
defaults_applied: dict[str, MigrationValue] = {}
|
|
659
|
-
|
|
660
|
-
fields_dropped = _drop_fields(config, drop_fields)
|
|
661
|
-
target_fields = _target_field_names(target_class)
|
|
662
|
-
|
|
663
|
-
default_values_map = dict(default_values) if default_values is not None else {}
|
|
664
|
-
default_fields_list = list(default_fields) if default_fields is not None else []
|
|
665
|
-
|
|
666
|
-
overlap = set(default_values_map) & set(default_fields_list)
|
|
667
|
-
if overlap:
|
|
444
|
+
)
|
|
445
|
+
if not matches:
|
|
446
|
+
raise FileNotFoundError(
|
|
447
|
+
f"migration: source not found for namespace={namespace} hash={furu_hash}"
|
|
448
|
+
)
|
|
449
|
+
if len(matches) > 1:
|
|
668
450
|
raise ValueError(
|
|
669
|
-
f"migration:
|
|
451
|
+
f"migration: multiple sources found for namespace={namespace} hash={furu_hash}"
|
|
670
452
|
)
|
|
453
|
+
return matches[0]
|
|
454
|
+
|
|
671
455
|
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
456
|
+
def _apply_transforms(
|
|
457
|
+
source_obj: dict[str, JsonValue],
|
|
458
|
+
*,
|
|
459
|
+
target_fields: tuple[str, ...],
|
|
460
|
+
rename_field: Mapping[str, str],
|
|
461
|
+
drop_field: Iterable[str],
|
|
462
|
+
default_field: Iterable[str],
|
|
463
|
+
set_field: Mapping[str, JsonValue],
|
|
464
|
+
target_class: _FuruClass,
|
|
465
|
+
) -> dict[str, JsonValue]:
|
|
466
|
+
fields = {k: v for k, v in source_obj.items() if k != "__class__"}
|
|
467
|
+
target_field_set = set(target_fields)
|
|
468
|
+
|
|
469
|
+
for old, new in rename_field.items():
|
|
470
|
+
if old not in fields:
|
|
677
471
|
raise ValueError(
|
|
678
|
-
f"migration:
|
|
472
|
+
f"migration: rename_field missing source field: {_format_fields([old])}"
|
|
679
473
|
)
|
|
680
|
-
|
|
681
|
-
if unknown:
|
|
474
|
+
if new in fields:
|
|
682
475
|
raise ValueError(
|
|
683
|
-
f"migration:
|
|
476
|
+
f"migration: rename_field target already exists: {_format_fields([new])}"
|
|
684
477
|
)
|
|
685
|
-
|
|
686
|
-
if default_fields_list:
|
|
687
|
-
conflicts = set(default_fields_list) & remaining_fields
|
|
688
|
-
if conflicts:
|
|
478
|
+
if new not in target_field_set:
|
|
689
479
|
raise ValueError(
|
|
690
|
-
f"migration:
|
|
480
|
+
f"migration: rename_field target not in schema: {_format_fields([new])}"
|
|
691
481
|
)
|
|
692
|
-
|
|
693
|
-
|
|
482
|
+
fields[new] = fields.pop(old)
|
|
483
|
+
|
|
484
|
+
for name in drop_field:
|
|
485
|
+
if name not in fields:
|
|
694
486
|
raise ValueError(
|
|
695
|
-
f"migration:
|
|
487
|
+
f"migration: drop_field missing source field: {_format_fields([name])}"
|
|
696
488
|
)
|
|
489
|
+
fields.pop(name)
|
|
697
490
|
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
if missing_defaults:
|
|
491
|
+
for name in default_field:
|
|
492
|
+
if name not in target_field_set:
|
|
701
493
|
raise ValueError(
|
|
702
|
-
f"migration:
|
|
494
|
+
f"migration: default_field not in schema: {_format_fields([name])}"
|
|
703
495
|
)
|
|
496
|
+
if name in fields:
|
|
497
|
+
continue
|
|
498
|
+
fields[name] = _serialize_value(_default_value_for_field(target_class, name))
|
|
704
499
|
|
|
705
|
-
for
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
missing_fields = sorted(set(target_fields) - config_keys)
|
|
716
|
-
if missing_fields:
|
|
717
|
-
raise ValueError(
|
|
718
|
-
f"migration: missing required fields for target class: {_format_fields(missing_fields)}"
|
|
719
|
-
)
|
|
720
|
-
extra_fields = sorted(config_keys - set(target_fields))
|
|
721
|
-
if extra_fields:
|
|
722
|
-
raise ValueError(
|
|
723
|
-
f"migration: extra fields present; use drop_fields to remove: {_format_fields(extra_fields)}"
|
|
724
|
-
)
|
|
725
|
-
|
|
726
|
-
config["__class__"] = to_namespace
|
|
727
|
-
to_config = _typed_config(config)
|
|
728
|
-
_typecheck_config(to_config)
|
|
729
|
-
|
|
730
|
-
to_hash = FuruSerializer.compute_hash(to_config)
|
|
731
|
-
to_ref = _build_target_ref(target_class, to_namespace, to_hash)
|
|
732
|
-
|
|
733
|
-
return MigrationCandidate(
|
|
734
|
-
from_ref=from_ref,
|
|
735
|
-
to_ref=to_ref,
|
|
736
|
-
to_namespace=to_namespace,
|
|
737
|
-
to_config=to_config,
|
|
738
|
-
defaults_applied=defaults_applied,
|
|
739
|
-
fields_dropped=fields_dropped,
|
|
740
|
-
missing_fields=missing_fields,
|
|
741
|
-
extra_fields=extra_fields,
|
|
742
|
-
)
|
|
743
|
-
|
|
500
|
+
for name, value in set_field.items():
|
|
501
|
+
if name not in target_field_set:
|
|
502
|
+
raise ValueError(
|
|
503
|
+
f"migration: set_field not in schema: {_format_fields([name])}"
|
|
504
|
+
)
|
|
505
|
+
if name in fields:
|
|
506
|
+
raise ValueError(
|
|
507
|
+
f"migration: set_field already set: {_format_fields([name])}"
|
|
508
|
+
)
|
|
509
|
+
fields[name] = _serialize_value(value)
|
|
744
510
|
|
|
745
|
-
def _drop_fields(
|
|
746
|
-
config: dict[str, JsonValue], drop_fields: Iterable[str] | None
|
|
747
|
-
) -> list[str]:
|
|
748
|
-
if drop_fields is None:
|
|
749
|
-
return []
|
|
750
|
-
fields = list(drop_fields)
|
|
751
|
-
unknown = [field for field in fields if field not in config]
|
|
752
|
-
if unknown:
|
|
753
|
-
raise ValueError(
|
|
754
|
-
f"migration: drop_fields contains unknown fields: {_format_fields(unknown)}"
|
|
755
|
-
)
|
|
756
|
-
for field in fields:
|
|
757
|
-
config.pop(field, None)
|
|
758
511
|
return fields
|
|
759
512
|
|
|
760
513
|
|
|
761
|
-
def
|
|
762
|
-
return [field.logical_name for field in chz.chz_fields(target_class).values()]
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
def _missing_class_defaults(
|
|
766
|
-
target_class: type[Furu],
|
|
767
|
-
default_fields: list[str],
|
|
768
|
-
) -> list[str]:
|
|
769
|
-
fields = chz.chz_fields(target_class)
|
|
770
|
-
missing: list[str] = []
|
|
771
|
-
for name in default_fields:
|
|
772
|
-
field = fields[name]
|
|
773
|
-
if field._default is not CHZ_MISSING:
|
|
774
|
-
continue
|
|
775
|
-
if not isinstance(field._default_factory, MISSING_TYPE):
|
|
776
|
-
continue
|
|
777
|
-
missing.append(name)
|
|
778
|
-
return missing
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
def _default_value_for_field(
|
|
782
|
-
target_class: type[Furu],
|
|
783
|
-
default_source: Furu | None,
|
|
784
|
-
field_name: str,
|
|
785
|
-
) -> MigrationValue:
|
|
786
|
-
if default_source is not None:
|
|
787
|
-
return getattr(default_source, field_name)
|
|
514
|
+
def _default_value_for_field(target_class: _FuruClass, name: str) -> JsonValue:
|
|
788
515
|
fields = chz.chz_fields(target_class)
|
|
789
|
-
|
|
516
|
+
fields_by_logical = {field.logical_name: field for field in fields.values()}
|
|
517
|
+
field = fields_by_logical.get(name)
|
|
518
|
+
if field is None:
|
|
519
|
+
raise ValueError(
|
|
520
|
+
f"migration: default_field missing defaults for fields: {_format_fields([name])}"
|
|
521
|
+
)
|
|
790
522
|
if field._default is not CHZ_MISSING:
|
|
791
523
|
return field._default
|
|
792
524
|
if not isinstance(field._default_factory, MISSING_TYPE):
|
|
793
525
|
return field._default_factory()
|
|
794
526
|
raise ValueError(
|
|
795
|
-
f"migration:
|
|
527
|
+
f"migration: default_field missing defaults for fields: {_format_fields([name])}"
|
|
796
528
|
)
|
|
797
529
|
|
|
798
530
|
|
|
799
|
-
def
|
|
800
|
-
values: Mapping[str, MigrationValue],
|
|
801
|
-
) -> dict[str, JsonValue] | None:
|
|
802
|
-
if not values:
|
|
803
|
-
return None
|
|
804
|
-
return {key: _serialize_value(value) for key, value in values.items()}
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
def _serialize_value(value: MigrationValue) -> JsonValue:
|
|
531
|
+
def _serialize_value(value: JsonValue) -> JsonValue:
|
|
808
532
|
result = FuruSerializer.to_dict(value)
|
|
809
533
|
if result is None:
|
|
810
534
|
return result
|
|
811
535
|
if isinstance(result, (str, int, float, bool, list, dict)):
|
|
812
536
|
return result
|
|
813
|
-
raise TypeError(f"
|
|
537
|
+
raise TypeError(f"migration: unsupported value type {type(result)}")
|
|
814
538
|
|
|
815
539
|
|
|
816
|
-
def
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
540
|
+
def _write_alias(
|
|
541
|
+
*,
|
|
542
|
+
target_obj: JsonValue,
|
|
543
|
+
original_ref: FuruRef,
|
|
544
|
+
target_ref: FuruRef,
|
|
545
|
+
source_ref: FuruRef,
|
|
546
|
+
skips: list[MigrationSkip],
|
|
547
|
+
conflict: MigrationConflict,
|
|
548
|
+
origin: str | None,
|
|
549
|
+
note: str | None,
|
|
550
|
+
) -> MigrationRecord | None:
|
|
551
|
+
try:
|
|
552
|
+
target_ref.directory.mkdir(parents=True, exist_ok=False)
|
|
553
|
+
except FileExistsError:
|
|
554
|
+
reason = "migration: target already exists"
|
|
555
|
+
if conflict == "skip":
|
|
556
|
+
skips.append(MigrationSkip(source=source_ref, reason=reason))
|
|
557
|
+
return None
|
|
558
|
+
raise ValueError(reason) from None
|
|
559
|
+
StateManager.ensure_internal_dir(target_ref.directory)
|
|
560
|
+
_write_migrated_state(target_ref.directory)
|
|
561
|
+
|
|
562
|
+
metadata = MetadataManager.create_metadata(
|
|
563
|
+
target_obj, target_ref.directory, ignore_diff=True
|
|
564
|
+
)
|
|
565
|
+
MetadataManager.write_metadata(metadata, target_ref.directory)
|
|
823
566
|
|
|
567
|
+
now = _dt.datetime.now(_dt.UTC).isoformat(timespec="seconds")
|
|
568
|
+
record = MigrationRecord(
|
|
569
|
+
kind="alias",
|
|
570
|
+
policy="alias",
|
|
571
|
+
from_namespace=original_ref.namespace,
|
|
572
|
+
from_hash=original_ref.furu_hash,
|
|
573
|
+
from_root=original_ref.root,
|
|
574
|
+
to_namespace=target_ref.namespace,
|
|
575
|
+
to_hash=target_ref.furu_hash,
|
|
576
|
+
to_root=target_ref.root,
|
|
577
|
+
migrated_at=now,
|
|
578
|
+
overwritten_at=None,
|
|
579
|
+
default_values=None,
|
|
580
|
+
origin=origin,
|
|
581
|
+
note=note,
|
|
582
|
+
)
|
|
583
|
+
MigrationManager.write_migration(record, target_ref.directory)
|
|
584
|
+
return record
|
|
824
585
|
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
586
|
+
|
|
587
|
+
def _write_migrated_state(directory: Path) -> None:
|
|
588
|
+
def mutate(state) -> None:
|
|
589
|
+
state.result = _StateResultMigrated(status="migrated")
|
|
590
|
+
state.attempt = None
|
|
591
|
+
|
|
592
|
+
StateManager.update_state(directory, mutate)
|
|
829
593
|
|
|
830
594
|
|
|
831
|
-
def
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
595
|
+
def _ensure_original_success(original_ref: FuruRef) -> None:
|
|
596
|
+
if not StateManager.success_marker_exists(original_ref.directory):
|
|
597
|
+
raise ValueError("migration: original artifact is not successful")
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _alias_schema_conflict(
|
|
601
|
+
alias_index: dict[tuple[str, str, RootKind], list[MigrationRecord]],
|
|
602
|
+
alias_schema_cache: dict[Path, tuple[str, ...]],
|
|
603
|
+
alias_key: tuple[str, str, RootKind],
|
|
604
|
+
target_schema_key: tuple[str, ...],
|
|
605
|
+
) -> bool:
|
|
606
|
+
records = alias_index.get(alias_key, [])
|
|
607
|
+
for record in records:
|
|
608
|
+
alias_dir = MigrationManager.resolve_dir(record, target="to")
|
|
609
|
+
if alias_dir not in alias_schema_cache:
|
|
610
|
+
metadata = MetadataManager.read_metadata_raw(alias_dir)
|
|
611
|
+
if metadata is None:
|
|
612
|
+
raise FileNotFoundError(
|
|
613
|
+
f"migration: metadata not found for alias {alias_dir}"
|
|
614
|
+
)
|
|
615
|
+
alias_schema_cache[alias_dir] = schema_key_from_metadata_raw(metadata)
|
|
616
|
+
if alias_schema_cache[alias_dir] == target_schema_key:
|
|
617
|
+
return True
|
|
618
|
+
return False
|
|
843
619
|
|
|
844
620
|
|
|
845
|
-
def
|
|
846
|
-
target_class: type[Furu],
|
|
621
|
+
def _refs_by_schema(
|
|
847
622
|
namespace: str,
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
623
|
+
schema_key: tuple[str, ...],
|
|
624
|
+
*,
|
|
625
|
+
match: bool,
|
|
626
|
+
) -> list[FuruRef]:
|
|
627
|
+
refs: list[FuruRef] = []
|
|
628
|
+
for ref, metadata in _iter_namespace_metadata(
|
|
629
|
+
namespace,
|
|
630
|
+
include_alias_sources=True,
|
|
631
|
+
):
|
|
632
|
+
current_key = schema_key_from_metadata_raw(metadata)
|
|
633
|
+
if (current_key == schema_key) == match:
|
|
634
|
+
refs.append(ref)
|
|
635
|
+
refs.sort(key=lambda item: item.furu_hash)
|
|
636
|
+
return refs
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _namespace_str(cls: _FuruClass) -> str:
|
|
640
|
+
return ".".join(cls._namespace().parts)
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def _target_ref(cls: _FuruClass, furu_hash: str) -> FuruRef:
|
|
644
|
+
root = FURU_CONFIG.get_root(version_controlled=cls.version_controlled)
|
|
645
|
+
namespace = _namespace_str(cls)
|
|
852
646
|
directory = root / Path(*namespace.split(".")) / furu_hash
|
|
647
|
+
root_kind: RootKind = "git" if cls.version_controlled else "data"
|
|
853
648
|
return FuruRef(
|
|
854
649
|
namespace=namespace,
|
|
855
650
|
furu_hash=furu_hash,
|
|
@@ -858,72 +653,5 @@ def _build_target_ref(
|
|
|
858
653
|
)
|
|
859
654
|
|
|
860
655
|
|
|
861
|
-
def _resolve_target_class(namespace: str) -> type[Furu]:
|
|
862
|
-
module_path, _, class_name = namespace.rpartition(".")
|
|
863
|
-
if not module_path:
|
|
864
|
-
raise ValueError(f"migration: unable to resolve target class: {namespace}")
|
|
865
|
-
try:
|
|
866
|
-
module = importlib.import_module(module_path)
|
|
867
|
-
except Exception as exc: # pragma: no cover - import errors
|
|
868
|
-
raise ValueError(
|
|
869
|
-
f"migration: unable to resolve target class: {namespace}"
|
|
870
|
-
) from exc
|
|
871
|
-
obj = getattr(module, class_name, None)
|
|
872
|
-
if obj is None:
|
|
873
|
-
raise ValueError(f"migration: unable to resolve target class: {namespace}")
|
|
874
|
-
if not _is_furu_class(obj):
|
|
875
|
-
raise ValueError(f"migration: unable to resolve target class: {namespace}")
|
|
876
|
-
return cast(type[Furu], obj)
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
def _is_furu_class(value: object) -> bool:
|
|
880
|
-
return isinstance(value, type) and issubclass(value, Furu)
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
def _namespace_str(target_class: type[Furu]) -> str:
|
|
884
|
-
return ".".join(target_class._namespace().parts)
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
def _extract_namespace(config: dict[str, JsonValue]) -> str:
|
|
888
|
-
class_name = config.get("__class__")
|
|
889
|
-
if isinstance(class_name, str):
|
|
890
|
-
return class_name
|
|
891
|
-
raise ValueError("migration: unable to resolve target class: <unknown>")
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
def _target_status(candidate: MigrationCandidate) -> str | None:
|
|
895
|
-
to_obj = FuruSerializer.from_dict(candidate.to_config)
|
|
896
|
-
state = to_obj.get_state(candidate.to_ref.directory)
|
|
897
|
-
if isinstance(state.result, _StateResultMigrated):
|
|
898
|
-
return None
|
|
899
|
-
if state.result.status == "success":
|
|
900
|
-
return "success"
|
|
901
|
-
attempt = state.attempt
|
|
902
|
-
if isinstance(attempt, _StateAttemptRunning):
|
|
903
|
-
return "running"
|
|
904
|
-
return None
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
def _expand_skip_keys(
|
|
908
|
-
conflicts: Iterable[_CandidateKey],
|
|
909
|
-
parent_map: dict[_CandidateKey, _CandidateKey | None],
|
|
910
|
-
) -> set[_CandidateKey]:
|
|
911
|
-
skipped = set(conflicts)
|
|
912
|
-
changed = True
|
|
913
|
-
while changed:
|
|
914
|
-
changed = False
|
|
915
|
-
for key, parent in parent_map.items():
|
|
916
|
-
if parent is None:
|
|
917
|
-
continue
|
|
918
|
-
if parent in skipped and key not in skipped:
|
|
919
|
-
skipped.add(key)
|
|
920
|
-
changed = True
|
|
921
|
-
return skipped
|
|
922
|
-
|
|
923
|
-
|
|
924
656
|
def _format_fields(fields: Iterable[str]) -> str:
|
|
925
657
|
return ", ".join(sorted(fields))
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
def _typed_config(config: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
|
929
|
-
return {str(key): value for key, value in config.items()}
|