pipelime-python 2.3.0__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipelime/__init__.py +1 -1
- pipelime/choixe/ast/nodes.py +41 -3
- pipelime/choixe/utils/common.py +35 -1
- pipelime/choixe/visitors/decoder.py +5 -2
- pipelime/choixe/visitors/inspector.py +8 -5
- pipelime/choixe/visitors/processor.py +19 -8
- pipelime/choixe/xconfig.py +18 -3
- pipelime/cli/main.py +16 -8
- pipelime/cli/pretty_print.py +72 -77
- pipelime/cli/tui/tui.py +3 -2
- pipelime/cli/tui/utils.py +27 -24
- pipelime/cli/utils.py +64 -37
- pipelime/commands/general.py +56 -53
- pipelime/commands/interfaces.py +92 -106
- pipelime/commands/piper.py +13 -19
- pipelime/commands/resume.py +9 -7
- pipelime/commands/shell.py +1 -3
- pipelime/commands/split_ops.py +15 -28
- pipelime/commands/tempman.py +2 -2
- pipelime/commands/toy_dataset.py +1 -3
- pipelime/piper/__init__.py +1 -0
- pipelime/piper/checkpoint.py +5 -3
- pipelime/piper/model.py +105 -80
- pipelime/piper/parsers/choixe_parser.py +1 -1
- pipelime/piper/progress/listener/receiver/zmq.py +1 -1
- pipelime/piper/progress/model.py +3 -3
- pipelime/piper/progress/tracker/zmq.py +1 -1
- pipelime/sequences/grabber.py +3 -2
- pipelime/sequences/pipes/base.py +2 -3
- pipelime/sequences/pipes/mapping.py +4 -3
- pipelime/sequences/pipes/operations.py +7 -14
- pipelime/sequences/pipes/validation.py +3 -5
- pipelime/sequences/pipes/writers.py +9 -7
- pipelime/sequences/sample.py +6 -2
- pipelime/sequences/samples_sequence.py +16 -17
- pipelime/sequences/sources/from_callable.py +1 -1
- pipelime/sequences/sources/raw.py +1 -1
- pipelime/sequences/sources/readers.py +22 -13
- pipelime/sequences/sources/toy_dataset.py +3 -2
- pipelime/sequences/utils.py +3 -8
- pipelime/stages/augmentations.py +30 -36
- pipelime/stages/base.py +33 -42
- pipelime/stages/entities.py +167 -99
- pipelime/stages/item_info.py +3 -2
- pipelime/stages/item_replacement.py +7 -8
- pipelime/stages/item_sources.py +1 -1
- pipelime/stages/key_transformations.py +3 -2
- pipelime/utils/pydantic_compat.py +1064 -0
- pipelime/utils/pydantic_types.py +186 -220
- {pipelime_python-2.3.0.dist-info → pipelime_python-3.0.1.dist-info}/METADATA +6 -2
- {pipelime_python-2.3.0.dist-info → pipelime_python-3.0.1.dist-info}/RECORD +54 -53
- {pipelime_python-2.3.0.dist-info → pipelime_python-3.0.1.dist-info}/WHEEL +1 -1
- {pipelime_python-2.3.0.dist-info → pipelime_python-3.0.1.dist-info}/entry_points.txt +0 -0
- {pipelime_python-2.3.0.dist-info → pipelime_python-3.0.1.dist-info}/licenses/LICENSE +0 -0
pipelime/__init__.py
CHANGED
pipelime/choixe/ast/nodes.py
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import functools
|
|
4
|
+
import types
|
|
5
|
+
import typing
|
|
3
6
|
from abc import ABC, abstractmethod
|
|
4
|
-
from dataclasses import field
|
|
7
|
+
from dataclasses import dataclass, field, fields
|
|
5
8
|
from typing import Any, Dict, Optional, Sequence, Tuple, Union
|
|
6
9
|
|
|
7
|
-
from pydantic.v1.dataclasses import dataclass
|
|
8
|
-
|
|
9
10
|
|
|
10
11
|
class NodeVisitor: # pragma: no cover
|
|
11
12
|
"""Base class for all Choixe visitors. A visitor can be seen as a special recursive
|
|
@@ -87,10 +88,47 @@ class NodeVisitor: # pragma: no cover
|
|
|
87
88
|
return self._ignore(node)
|
|
88
89
|
|
|
89
90
|
|
|
91
|
+
@functools.lru_cache(maxsize=None)
|
|
92
|
+
def _node_typed_fields(cls: type) -> Tuple[Tuple[str, Tuple[type, ...], bool], ...]:
|
|
93
|
+
"""For each dataclass field of `cls` whose resolved annotation is a `Node`
|
|
94
|
+
subclass, `Optional[<Node subclass>]`, or a `Union` of `Node` subclasses,
|
|
95
|
+
returns `(field_name, allowed_node_types, allow_none)`. Container-typed
|
|
96
|
+
fields (e.g. `Dict[...]`, `Sequence[...]`) and `Any` are left out. Cached
|
|
97
|
+
per class (outside any instance, since `HashNode.__hash__` hashes
|
|
98
|
+
`self.__dict__.values()`)."""
|
|
99
|
+
hints = typing.get_type_hints(cls)
|
|
100
|
+
specs = []
|
|
101
|
+
for f in fields(cls):
|
|
102
|
+
hint = hints.get(f.name)
|
|
103
|
+
is_union = typing.get_origin(hint) is Union or isinstance(hint, types.UnionType)
|
|
104
|
+
candidates = typing.get_args(hint) if is_union else (hint,)
|
|
105
|
+
node_types = tuple(
|
|
106
|
+
c for c in candidates if isinstance(c, type) and issubclass(c, Node)
|
|
107
|
+
)
|
|
108
|
+
if node_types:
|
|
109
|
+
specs.append((f.name, node_types, type(None) in candidates))
|
|
110
|
+
return tuple(specs)
|
|
111
|
+
|
|
112
|
+
|
|
90
113
|
@dataclass
|
|
91
114
|
class Node(ABC):
|
|
92
115
|
"""A generic element of the Choixe AST, all nodes must implement this interface."""
|
|
93
116
|
|
|
117
|
+
def __post_init__(self) -> None:
|
|
118
|
+
for name, node_types, allow_none in _node_typed_fields(type(self)):
|
|
119
|
+
value = getattr(self, name, None)
|
|
120
|
+
if value is None and allow_none:
|
|
121
|
+
continue
|
|
122
|
+
if isinstance(value, node_types):
|
|
123
|
+
continue
|
|
124
|
+
expected = " or ".join(t.__name__ for t in node_types)
|
|
125
|
+
if allow_none:
|
|
126
|
+
expected += " or None"
|
|
127
|
+
raise TypeError(
|
|
128
|
+
f"{type(self).__name__}.{name} expected {expected}, "
|
|
129
|
+
f"got {type(value).__name__}"
|
|
130
|
+
)
|
|
131
|
+
|
|
94
132
|
@abstractmethod
|
|
95
133
|
def accept(self, visitor: NodeVisitor) -> Any:
|
|
96
134
|
"""Accepts an incoming visitor. This method should simply call the respective
|
pipelime/choixe/utils/common.py
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
|
+
import re
|
|
1
2
|
import typing as t
|
|
2
3
|
|
|
4
|
+
# pydash's string path delimiter (the same in pydash 8.0 and 8.1): an unescaped `.`,
|
|
5
|
+
# or a `[<int>]` list index (captured, so that it is kept as a path part)
|
|
6
|
+
_PYDASH_PATH_DELIM = re.compile(r"(?<!\\)(?:\\\\)*\.|(\[-?\d+\])")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def pydash_path(key_path: t.Any) -> t.Any:
|
|
10
|
+
r"""A key path for pydash (``get``, ``set_``, ``has``, ``to_path``) addressing the
|
|
11
|
+
same value on every pydash version.
|
|
12
|
+
|
|
13
|
+
pydash 8.1 reads the empty key of a leading, trailing or doubled ``.`` in a string
|
|
14
|
+
path (``".a"`` is ``["", "a"]``, ``"."`` is ``["", ""]``), where earlier versions
|
|
15
|
+
dropped it (``["a"]``, ``[]``: the whole object). pipelime keeps the pre-8.1 meaning:
|
|
16
|
+
the empty keys of a dotted or indexed string path are dropped here, and the other
|
|
17
|
+
parts, escapes included, are joined back into a canonical path (``"a.b[0].c"``) that
|
|
18
|
+
every pydash version splits the same way. A path left with a single key is returned
|
|
19
|
+
unescaped (pydash takes a plain key as is), with no key as ``[]``. Any other path (a
|
|
20
|
+
plain key without ``.`` or ``[``, a key list, an int) is returned unchanged.
|
|
21
|
+
"""
|
|
22
|
+
if not (isinstance(key_path, str) and ("." in key_path or "[" in key_path)):
|
|
23
|
+
return key_path
|
|
24
|
+
# `re.split` alternates the text between two delimiters (a key, "" if empty) and
|
|
25
|
+
# the captured group (a `[<int>]` list index, None for a `.`)
|
|
26
|
+
parts = _PYDASH_PATH_DELIM.split(key_path)
|
|
27
|
+
path = ""
|
|
28
|
+
for i, part in enumerate(parts):
|
|
29
|
+
if part:
|
|
30
|
+
path += part if i % 2 or not path else "." + part
|
|
31
|
+
if not path:
|
|
32
|
+
return []
|
|
33
|
+
if "." in path or "[" in path:
|
|
34
|
+
return path
|
|
35
|
+
return path.replace("\\\\", "\\") # pydash's unescape; no `\.` is left here
|
|
36
|
+
|
|
3
37
|
|
|
4
38
|
class _RefItem:
|
|
5
39
|
def __init__(self, parent, idx):
|
|
@@ -73,7 +107,7 @@ def deep_set_( # noqa: C901
|
|
|
73
107
|
from pydash.utilities import to_path
|
|
74
108
|
|
|
75
109
|
if key_path is not None: # pragma: no branch
|
|
76
|
-
key_path_tokens = to_path(key_path)
|
|
110
|
+
key_path_tokens = to_path(pydash_path(key_path))
|
|
77
111
|
parent_node = _RefItem(obj, None)
|
|
78
112
|
for tk in key_path_tokens:
|
|
79
113
|
if isinstance(tk, int):
|
|
@@ -3,7 +3,7 @@ from pathlib import Path
|
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
6
|
-
from pydantic
|
|
6
|
+
from pydantic import BaseModel
|
|
7
7
|
|
|
8
8
|
from pipelime.choixe.ast.nodes import LiteralNode, Node
|
|
9
9
|
from pipelime.choixe.visitors.unparser import Unparser
|
|
@@ -20,7 +20,10 @@ class Decoder(Unparser):
|
|
|
20
20
|
return data.item()
|
|
21
21
|
elif isinstance(data, BaseModel):
|
|
22
22
|
symbol = f"{data.__module__}.{data.__class__.__qualname__}"
|
|
23
|
-
return {
|
|
23
|
+
return {
|
|
24
|
+
"$model": symbol,
|
|
25
|
+
"$args": json.loads(data.model_dump_json(by_alias=True)),
|
|
26
|
+
}
|
|
24
27
|
elif isinstance(data, Path):
|
|
25
28
|
return str(data)
|
|
26
29
|
else:
|
|
@@ -10,6 +10,7 @@ import pydash as py_
|
|
|
10
10
|
|
|
11
11
|
import pipelime.choixe.ast.nodes as ast
|
|
12
12
|
from pipelime.choixe.ast.parser import parse
|
|
13
|
+
from pipelime.choixe.utils.common import pydash_path
|
|
13
14
|
from pipelime.choixe.utils.io import load
|
|
14
15
|
|
|
15
16
|
|
|
@@ -87,11 +88,11 @@ class Inspector(ast.NodeVisitor):
|
|
|
87
88
|
|
|
88
89
|
id_ = node.identifier.data # type: ignore
|
|
89
90
|
default = None if node.default is None else unparse(node.default)
|
|
90
|
-
variables = py_.set_({}, id_, default)
|
|
91
|
+
variables = py_.set_({}, pydash_path(id_), default)
|
|
91
92
|
|
|
92
93
|
help_strings = {}
|
|
93
94
|
if node.help is not None:
|
|
94
|
-
help_strings = py_.set_({}, id_, node.help.data)
|
|
95
|
+
help_strings = py_.set_({}, pydash_path(id_), node.help.data)
|
|
95
96
|
|
|
96
97
|
environ = {}
|
|
97
98
|
if node.env is not None and node.env.data: # type: ignore
|
|
@@ -150,7 +151,7 @@ class Inspector(ast.NodeVisitor):
|
|
|
150
151
|
if node.identifier is not None: # pragma: no branch
|
|
151
152
|
self._named_for_loops[node.identifier.data] = node.iterable.data
|
|
152
153
|
iterable_insp = (
|
|
153
|
-
Inspection(variables=py_.set_({}, node.iterable.data, None))
|
|
154
|
+
Inspection(variables=py_.set_({}, pydash_path(node.iterable.data), None))
|
|
154
155
|
if isinstance(node.iterable.data, str)
|
|
155
156
|
else Inspection(processed=True)
|
|
156
157
|
)
|
|
@@ -160,7 +161,9 @@ class Inspector(ast.NodeVisitor):
|
|
|
160
161
|
def visit_switch(self, node: ast.SwitchNode) -> Inspection:
|
|
161
162
|
insp = node.value.accept(self)
|
|
162
163
|
if insp.processed: # pragma: no branch
|
|
163
|
-
insp = Inspection(
|
|
164
|
+
insp = Inspection(
|
|
165
|
+
variables=py_.set_({}, pydash_path(node.value.data), None) # type: ignore
|
|
166
|
+
)
|
|
164
167
|
cases_insp = sum(
|
|
165
168
|
[x.accept(self) + y.accept(self) for x, y in node.cases],
|
|
166
169
|
start=Inspection(processed=True),
|
|
@@ -186,7 +189,7 @@ class Inspector(ast.NodeVisitor):
|
|
|
186
189
|
if key:
|
|
187
190
|
iterable_name = self._named_for_loops[loop_id]
|
|
188
191
|
full_path = f"{iterable_name}.{key}"
|
|
189
|
-
py_.set_(variables, full_path, None)
|
|
192
|
+
py_.set_(variables, pydash_path(full_path), None)
|
|
190
193
|
insp = insp + sub_insp + Inspection(variables=variables)
|
|
191
194
|
return insp
|
|
192
195
|
|
|
@@ -11,6 +11,7 @@ import pydash as py_
|
|
|
11
11
|
|
|
12
12
|
import pipelime.choixe.ast.nodes as ast
|
|
13
13
|
from pipelime.choixe.ast.parser import parse
|
|
14
|
+
from pipelime.choixe.utils.common import pydash_path
|
|
14
15
|
from pipelime.choixe.utils.imports import import_symbol
|
|
15
16
|
from pipelime.choixe.utils.io import PipelimeTmp, load
|
|
16
17
|
from pipelime.choixe.utils.rand import rand
|
|
@@ -148,8 +149,8 @@ class Processor(ast.NodeVisitor):
|
|
|
148
149
|
var_value = None
|
|
149
150
|
found = False
|
|
150
151
|
|
|
151
|
-
if py_.has(self._context, id_):
|
|
152
|
-
var_value = py_.get(self._context, id_)
|
|
152
|
+
if py_.has(self._context, pydash_path(id_)):
|
|
153
|
+
var_value = py_.get(self._context, pydash_path(id_))
|
|
153
154
|
found = True
|
|
154
155
|
if not found and env:
|
|
155
156
|
value = os.getenv(id_)
|
|
@@ -214,14 +215,24 @@ class Processor(ast.NodeVisitor):
|
|
|
214
215
|
symbol_branches = node.symbol.accept(self)
|
|
215
216
|
args_branches = node.args.accept(self)
|
|
216
217
|
branches = self._branches(symbol_branches, args_branches)
|
|
217
|
-
|
|
218
|
+
models = []
|
|
219
|
+
for s, a in branches:
|
|
220
|
+
model_cls = import_symbol(s, cwd=self._cwd)
|
|
221
|
+
if not hasattr(model_cls, "model_validate"):
|
|
222
|
+
raise TypeError(
|
|
223
|
+
f"`$model` needs a pydantic v2 model, got {model_cls!r}: "
|
|
224
|
+
"pipelime 3 no longer supports pydantic.v1 models "
|
|
225
|
+
"(see docs/migration/pydantic_v2.md)"
|
|
226
|
+
)
|
|
227
|
+
models.append(model_cls.model_validate(a))
|
|
228
|
+
return models
|
|
218
229
|
|
|
219
230
|
def visit_for(self, node: ast.ForNode) -> List[Any]:
|
|
220
231
|
if isinstance(node.iterable.data, str):
|
|
221
|
-
if not py_.has(self._context, node.iterable.data):
|
|
232
|
+
if not py_.has(self._context, pydash_path(node.iterable.data)):
|
|
222
233
|
iterable = self._handle_missing_for(node.iterable.data)
|
|
223
234
|
else:
|
|
224
|
-
iterable = py_.get(self._context, node.iterable.data)
|
|
235
|
+
iterable = py_.get(self._context, pydash_path(node.iterable.data))
|
|
225
236
|
else:
|
|
226
237
|
iterable = node.iterable.data
|
|
227
238
|
|
|
@@ -278,10 +289,10 @@ class Processor(ast.NodeVisitor):
|
|
|
278
289
|
for branch in all_branches:
|
|
279
290
|
varname = branch[0]
|
|
280
291
|
|
|
281
|
-
if not py_.has(self._context, varname):
|
|
292
|
+
if not py_.has(self._context, pydash_path(varname)):
|
|
282
293
|
value = self._handle_missing_switch(varname)
|
|
283
294
|
else:
|
|
284
|
-
value = py_.get(self._context, varname)
|
|
295
|
+
value = py_.get(self._context, pydash_path(varname))
|
|
285
296
|
|
|
286
297
|
# Match the value to the correct case
|
|
287
298
|
for i in range(len(node.cases)):
|
|
@@ -321,7 +332,7 @@ class Processor(ast.NodeVisitor):
|
|
|
321
332
|
items = []
|
|
322
333
|
for branch in branches:
|
|
323
334
|
loop_id, _, key = str(branch).partition(sep) # type: ignore
|
|
324
|
-
item = py_.get(self._loop_data[loop_id].item, f"{sep}{key}")
|
|
335
|
+
item = py_.get(self._loop_data[loop_id].item, pydash_path(f"{sep}{key}"))
|
|
325
336
|
items.append(item)
|
|
326
337
|
return items
|
|
327
338
|
|
pipelime/choixe/xconfig.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from copy import deepcopy
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
|
|
5
6
|
|
|
@@ -9,7 +10,7 @@ from schema import Schema
|
|
|
9
10
|
|
|
10
11
|
from pipelime.choixe.ast.nodes import Node
|
|
11
12
|
from pipelime.choixe.ast.parser import parse
|
|
12
|
-
from pipelime.choixe.utils.common import deep_set_
|
|
13
|
+
from pipelime.choixe.utils.common import deep_set_, pydash_path
|
|
13
14
|
from pipelime.choixe.utils.io import dump, load
|
|
14
15
|
from pipelime.choixe.visitors import Inspection, decode, inspect, process, walk
|
|
15
16
|
|
|
@@ -38,6 +39,11 @@ class XConfig(Box):
|
|
|
38
39
|
validation. Defaults to None.
|
|
39
40
|
"""
|
|
40
41
|
|
|
42
|
+
# Initialize the box config. Nested containers must be plain `Box` objects:
|
|
43
|
+
# `XConfig.__init__` does not accept box options, and recent python-box
|
|
44
|
+
# versions would otherwise default `box_class` to the subclass itself.
|
|
45
|
+
super().__init__(box_class=Box)
|
|
46
|
+
|
|
41
47
|
# options
|
|
42
48
|
self._cwd = cwd
|
|
43
49
|
self._schema = None
|
|
@@ -93,6 +99,15 @@ class XConfig(Box):
|
|
|
93
99
|
"""
|
|
94
100
|
return XConfig(data=self.decode(), cwd=self.get_cwd(), schema=self.get_schema())
|
|
95
101
|
|
|
102
|
+
def __deepcopy__(self, memo: Optional[Dict[int, Any]] = None) -> XConfig:
|
|
103
|
+
# `Box.__deepcopy__` would build the copy passing box options to
|
|
104
|
+
# `XConfig.__init__` (or return a plain `Box`, on older python-box versions)
|
|
105
|
+
return XConfig(
|
|
106
|
+
data=deepcopy(self.to_dict(), memo),
|
|
107
|
+
cwd=self.get_cwd(),
|
|
108
|
+
schema=self.get_schema(),
|
|
109
|
+
)
|
|
110
|
+
|
|
96
111
|
def validate(self, replace: bool = True):
|
|
97
112
|
"""Validate internal schema, if any
|
|
98
113
|
|
|
@@ -139,7 +154,7 @@ class XConfig(Box):
|
|
|
139
154
|
Returns:
|
|
140
155
|
Any: The value at the specified path.
|
|
141
156
|
"""
|
|
142
|
-
return py_.get(self, full_key, default=default)
|
|
157
|
+
return py_.get(self, pydash_path(full_key), default=default)
|
|
143
158
|
|
|
144
159
|
def deep_set(
|
|
145
160
|
self,
|
|
@@ -159,7 +174,7 @@ class XConfig(Box):
|
|
|
159
174
|
Defaults to False.
|
|
160
175
|
"""
|
|
161
176
|
|
|
162
|
-
if not only_valid_keys or py_.has(self, full_key):
|
|
177
|
+
if not only_valid_keys or py_.has(self, pydash_path(full_key)):
|
|
163
178
|
deep_set_(self, key_path=full_key, value=value, append=append_values)
|
|
164
179
|
|
|
165
180
|
def deep_update(
|
pipelime/cli/main.py
CHANGED
|
@@ -6,7 +6,6 @@ import typing as t
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
|
|
8
8
|
import typer
|
|
9
|
-
from pydantic.v1 import BaseModel
|
|
10
9
|
|
|
11
10
|
from pipelime.choixe import XConfig
|
|
12
11
|
from pipelime.choixe.visitors.processor_ui import ProcessorUi
|
|
@@ -18,12 +17,15 @@ from pipelime.cli.utils import (
|
|
|
18
17
|
print_command_op_stage_info,
|
|
19
18
|
print_commands_ops_stages_list,
|
|
20
19
|
)
|
|
20
|
+
from pipelime.utils.pydantic_compat import PipelimeModel
|
|
21
21
|
|
|
22
22
|
if t.TYPE_CHECKING:
|
|
23
23
|
from pipelime.piper.checkpoint import Checkpoint
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
class PlCliOptions(
|
|
26
|
+
class PlCliOptions(PipelimeModel):
|
|
27
|
+
# a `PipelimeModel` for the v1 Optional rule: the options saved in a checkpoint
|
|
28
|
+
# may lack an `Optional` key
|
|
27
29
|
_namespace: t.ClassVar[str] = "__plmain"
|
|
28
30
|
|
|
29
31
|
config: t.List[Path]
|
|
@@ -43,7 +45,7 @@ class PlCliOptions(BaseModel):
|
|
|
43
45
|
pipelime_tmp: t.Optional[str]
|
|
44
46
|
|
|
45
47
|
def purged_dict(self):
|
|
46
|
-
return self._purge(self.
|
|
48
|
+
return self._purge(self.model_dump())
|
|
47
49
|
|
|
48
50
|
def _purge(self, data):
|
|
49
51
|
if isinstance(data, Path):
|
|
@@ -918,14 +920,19 @@ def run_command(
|
|
|
918
920
|
|
|
919
921
|
import time
|
|
920
922
|
|
|
921
|
-
from pydantic
|
|
923
|
+
from pydantic import ValidationError
|
|
924
|
+
from rich.markup import escape
|
|
922
925
|
|
|
923
926
|
from pipelime.choixe.utils.io import PipelimeTmp, dump
|
|
924
|
-
from pipelime.cli.pretty_print import
|
|
927
|
+
from pipelime.cli.pretty_print import (
|
|
928
|
+
print_command_outputs,
|
|
929
|
+
print_error,
|
|
930
|
+
print_info,
|
|
931
|
+
)
|
|
925
932
|
from pipelime.cli.tui import TuiApp, is_tui_needed
|
|
926
933
|
from pipelime.cli.utils import (
|
|
934
|
+
format_validation_error,
|
|
927
935
|
get_pipelime_command_cls,
|
|
928
|
-
show_field_alias_valerr,
|
|
929
936
|
time_to_str,
|
|
930
937
|
)
|
|
931
938
|
from pipelime.commands import TempCommand
|
|
@@ -972,12 +979,13 @@ def run_command(
|
|
|
972
979
|
# so let's show the tui again if it was needed in the first place
|
|
973
980
|
|
|
974
981
|
except ValidationError as e:
|
|
975
|
-
|
|
982
|
+
# plain text, not Rich markup: `[type=...]` and bracketed values must survive
|
|
983
|
+
print_error(escape(format_validation_error(e, cmd_cls)))
|
|
976
984
|
raise e
|
|
977
985
|
|
|
978
986
|
if verbose > 0:
|
|
979
987
|
print_info(f"\nCreated command `{command}`:")
|
|
980
|
-
print_info(cmd_obj.
|
|
988
|
+
print_info(cmd_obj.model_dump(), pretty=True)
|
|
981
989
|
|
|
982
990
|
if dry_run or verbose > 0:
|
|
983
991
|
print_info(f"\nRunning `{command}`...")
|
pipelime/cli/pretty_print.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import typing as t
|
|
3
3
|
|
|
4
|
-
from pydantic
|
|
4
|
+
from pydantic import BaseModel
|
|
5
5
|
from rich import box, get_console
|
|
6
6
|
from rich import print as rprint
|
|
7
7
|
from rich.markup import escape
|
|
@@ -9,6 +9,14 @@ from rich.panel import Panel
|
|
|
9
9
|
from rich.pretty import Pretty
|
|
10
10
|
from rich.table import Column, Table
|
|
11
11
|
|
|
12
|
+
from pipelime.utils.pydantic_compat import (
|
|
13
|
+
FieldView,
|
|
14
|
+
iter_fields,
|
|
15
|
+
model_title,
|
|
16
|
+
strip_annotated,
|
|
17
|
+
type_info,
|
|
18
|
+
)
|
|
19
|
+
|
|
12
20
|
if t.TYPE_CHECKING:
|
|
13
21
|
from pipelime.cli.utils import ActionInfo
|
|
14
22
|
|
|
@@ -92,9 +100,7 @@ def show_spinning_status(text: str):
|
|
|
92
100
|
|
|
93
101
|
|
|
94
102
|
def get_model_title(model_cls: t.Type[BaseModel]) -> str:
|
|
95
|
-
|
|
96
|
-
return model_cls.__config__.title
|
|
97
|
-
return model_cls.__name__
|
|
103
|
+
return model_title(model_cls)
|
|
98
104
|
|
|
99
105
|
|
|
100
106
|
def get_model_classpath(model_cls: t.Type[BaseModel]) -> str:
|
|
@@ -104,17 +110,15 @@ def get_model_classpath(model_cls: t.Type[BaseModel]) -> str:
|
|
|
104
110
|
|
|
105
111
|
|
|
106
112
|
def print_model_field_values(
|
|
107
|
-
model_fields: t.Mapping,
|
|
113
|
+
model_fields: t.Mapping[str, FieldView],
|
|
108
114
|
port_values: t.Mapping[str, t.Any],
|
|
109
115
|
icon: str = "",
|
|
110
116
|
):
|
|
111
117
|
for k, v in port_values.items():
|
|
112
118
|
rprint(f"\n{icon if icon else '***'} {k}:")
|
|
113
119
|
# Ports might be virtual, as in ShellCommand, so they might not be in the model
|
|
114
|
-
if k in model_fields and model_fields[k].
|
|
115
|
-
rprint(
|
|
116
|
-
f"[italic grey50]{escape(model_fields[k].field_info.description)}[/]"
|
|
117
|
-
)
|
|
120
|
+
if k in model_fields and model_fields[k].description:
|
|
121
|
+
rprint(f"[italic grey50]{escape(model_fields[k].description)}[/]")
|
|
118
122
|
rprint(
|
|
119
123
|
"[green]"
|
|
120
124
|
+ escape(str(v) if isinstance(v, (bytes, str)) else repr(v))
|
|
@@ -122,12 +126,20 @@ def print_model_field_values(
|
|
|
122
126
|
)
|
|
123
127
|
|
|
124
128
|
|
|
129
|
+
def _fields_by_name(model_cls) -> t.Dict[str, FieldView]:
|
|
130
|
+
return {f.name: f for f in iter_fields(model_cls)}
|
|
131
|
+
|
|
132
|
+
|
|
125
133
|
def print_command_inputs(command: "PipelimeCommand"): # type: ignore # noqa: E602,F821
|
|
126
|
-
print_model_field_values(
|
|
134
|
+
print_model_field_values(
|
|
135
|
+
_fields_by_name(type(command)), command.get_inputs(), _input_icon()
|
|
136
|
+
)
|
|
127
137
|
|
|
128
138
|
|
|
129
139
|
def print_command_outputs(command: "PipelimeCommand"): # type: ignore # noqa: E602,F821
|
|
130
|
-
print_model_field_values(
|
|
140
|
+
print_model_field_values(
|
|
141
|
+
_fields_by_name(type(command)), command.get_outputs(), _output_icon()
|
|
142
|
+
)
|
|
131
143
|
|
|
132
144
|
|
|
133
145
|
def print_actions_short_help(*actions_info: "ActionInfo", show_class_path: bool = True):
|
|
@@ -218,7 +230,7 @@ def print_model_info(
|
|
|
218
230
|
|
|
219
231
|
def _field_row(
|
|
220
232
|
grid: Table,
|
|
221
|
-
field,
|
|
233
|
+
field: FieldView,
|
|
222
234
|
indent: int,
|
|
223
235
|
indent_offs: int,
|
|
224
236
|
show_piper_port: bool,
|
|
@@ -227,35 +239,29 @@ def _field_row(
|
|
|
227
239
|
):
|
|
228
240
|
from enum import Enum
|
|
229
241
|
|
|
230
|
-
expand_help = field.
|
|
242
|
+
expand_help = field.extra.get("expand_help", False)
|
|
231
243
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
)
|
|
244
|
+
inner_type = field.inner_type
|
|
245
|
+
is_model = field.is_model and not inspect.isabstract(inner_type)
|
|
235
246
|
|
|
236
247
|
if show_description:
|
|
237
|
-
# NB: docs should not come from the inner
|
|
238
|
-
if field.
|
|
239
|
-
field_docs = field.
|
|
240
|
-
elif hasattr(
|
|
241
|
-
field_docs = str(inspect.getdoc(
|
|
248
|
+
# NB: docs should not come from the inner root type
|
|
249
|
+
if field.description:
|
|
250
|
+
field_docs = field.description
|
|
251
|
+
elif hasattr(inner_type, "__doc__") and inner_type.__doc__:
|
|
252
|
+
field_docs = str(inspect.getdoc(inner_type))
|
|
242
253
|
else:
|
|
243
254
|
field_docs = ""
|
|
244
255
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
field_outer_type = (
|
|
249
|
-
field.outer_type_.__fields__["__root__"].outer_type_
|
|
250
|
-
if has_root_item
|
|
251
|
-
else field.outer_type_
|
|
252
|
-
)
|
|
256
|
+
root_type = field.root_type if is_model else None
|
|
257
|
+
has_root_item = root_type is not None
|
|
258
|
+
field_outer_type = root_type if has_root_item else inner_type
|
|
253
259
|
|
|
254
260
|
if show_piper_port:
|
|
255
261
|
from pipelime.piper import PiperPortType
|
|
256
262
|
|
|
257
263
|
fport = str(
|
|
258
|
-
field.
|
|
264
|
+
field.extra.get("piper_port", PiperPortType.PARAMETER).value
|
|
259
265
|
).upper()
|
|
260
266
|
|
|
261
267
|
if fport == PiperPortType.INPUT.value.upper():
|
|
@@ -271,10 +277,10 @@ def _field_row(
|
|
|
271
277
|
+ ("[bold dark_orange]" if indent == 0 else "")
|
|
272
278
|
+ (
|
|
273
279
|
f"{escape(field.name)} / "
|
|
274
|
-
if field.
|
|
280
|
+
if field.populate_by_name and field.has_alias
|
|
275
281
|
else ""
|
|
276
282
|
)
|
|
277
|
-
+ f"{escape(field.
|
|
283
|
+
+ f"{escape(field.effective_alias)}"
|
|
278
284
|
+ ("[/]" if indent == 0 else "")
|
|
279
285
|
]
|
|
280
286
|
|
|
@@ -294,10 +300,13 @@ def _field_row(
|
|
|
294
300
|
line.append(fport) # type: ignore
|
|
295
301
|
|
|
296
302
|
# Default value
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
303
|
+
if field.required:
|
|
304
|
+
line.append("[red]✗[/]")
|
|
305
|
+
else:
|
|
306
|
+
field_default = field.default
|
|
307
|
+
if isinstance(field_default, Enum):
|
|
308
|
+
field_default = field_default.value
|
|
309
|
+
line.append(f"[green]{field_default}[/]")
|
|
301
310
|
|
|
302
311
|
grid.add_row(*line)
|
|
303
312
|
|
|
@@ -381,12 +390,9 @@ def _get_signature(model_cls: t.Type[BaseModel]) -> str:
|
|
|
381
390
|
|
|
382
391
|
return f"\n {formatted}"
|
|
383
392
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
for mfield in model_cls.__fields__.values()
|
|
388
|
-
if mfield.field_info.exclude
|
|
389
|
-
]
|
|
393
|
+
fields = list(iter_fields(model_cls))
|
|
394
|
+
fullname = {f.effective_alias: f.name for f in fields}
|
|
395
|
+
excluded = [f.effective_alias for f in fields if f.exclude]
|
|
390
396
|
|
|
391
397
|
sig = inspect.signature(model_cls)
|
|
392
398
|
sig = sig.replace(
|
|
@@ -418,48 +424,37 @@ def _is_model(type_):
|
|
|
418
424
|
return inspect.isclass(type_) and issubclass(type_, BaseModel)
|
|
419
425
|
|
|
420
426
|
|
|
427
|
+
_TYPING_MODULES = ("typing", "typing_extensions")
|
|
428
|
+
|
|
429
|
+
|
|
421
430
|
def _human_readable_type(field_outer_type):
|
|
422
431
|
from enum import Enum
|
|
423
432
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
get_origin,
|
|
428
|
-
is_union,
|
|
429
|
-
typing_base, # noqa: F401 # type: ignore
|
|
430
|
-
)
|
|
433
|
+
# constrained types (`Annotated[int, Gt(gt=0)]`) are shown as their base type
|
|
434
|
+
v = strip_annotated(field_outer_type)
|
|
435
|
+
ti = type_info(v)
|
|
431
436
|
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
if is_union(v_orig):
|
|
443
|
-
return " | ".join(map(_human_readable_type, get_args(v)))
|
|
444
|
-
if inspect.isclass(v_orig):
|
|
445
|
-
if issubclass(dict, v_orig):
|
|
446
|
-
return "{" + ": ".join(map(_human_readable_type, get_args(v))) + "}"
|
|
447
|
-
if issubclass(list, v_orig):
|
|
448
|
-
return "[" + ", ".join(map(_human_readable_type, get_args(v))) + ", ...]"
|
|
449
|
-
if issubclass(tuple, v_orig):
|
|
450
|
-
return "(" + ", ".join(map(_human_readable_type, get_args(v))) + ")"
|
|
437
|
+
if ti.is_union:
|
|
438
|
+
return " | ".join(map(_human_readable_type, ti.args))
|
|
439
|
+
if inspect.isclass(ti.origin):
|
|
440
|
+
if issubclass(dict, ti.origin):
|
|
441
|
+
return "{" + ": ".join(map(_human_readable_type, ti.args)) + "}"
|
|
442
|
+
if issubclass(list, ti.origin):
|
|
443
|
+
return "[" + ", ".join(map(_human_readable_type, ti.args)) + ", ...]"
|
|
444
|
+
if issubclass(tuple, ti.origin):
|
|
445
|
+
return "(" + ", ".join(map(_human_readable_type, ti.args)) + ")"
|
|
451
446
|
|
|
452
447
|
if inspect.isclass(v) and issubclass(v, Enum):
|
|
453
448
|
v = v.__name__ + "{" + ", ".join(e.name.lower() for e in v) + "}"
|
|
454
|
-
elif
|
|
455
|
-
#
|
|
449
|
+
elif ti.origin is not None:
|
|
450
|
+
# generic aliases (`Sequence[int]`, `Literal[...]`)
|
|
456
451
|
v = str(v).replace("typing.", "")
|
|
457
452
|
else:
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
453
|
+
if not inspect.isclass(v) and type(v).__module__ not in _TYPING_MODULES:
|
|
454
|
+
# a value among the type args, e.g. `...` in `Tuple[int, ...]`
|
|
455
|
+
v = type(v)
|
|
456
|
+
# classes and named typing objects (`TypeVar`) by name, the others as written
|
|
457
|
+
v = getattr(v, "__name__", None) or str(v).replace("typing.", "")
|
|
463
458
|
|
|
464
459
|
return v.replace("NoneType", "None")
|
|
465
460
|
|
|
@@ -497,8 +492,8 @@ def _iterate_model_fields(
|
|
|
497
492
|
add_blank_row,
|
|
498
493
|
):
|
|
499
494
|
no_data = True
|
|
500
|
-
for field in model_cls
|
|
501
|
-
if not field.
|
|
495
|
+
for field in iter_fields(model_cls):
|
|
496
|
+
if not field.exclude:
|
|
502
497
|
no_data = False
|
|
503
498
|
_field_row(
|
|
504
499
|
grid,
|
pipelime/cli/tui/tui.py
CHANGED
|
@@ -20,6 +20,7 @@ from pipelime.cli.tui.utils import (
|
|
|
20
20
|
from pipelime.cli.utils import parse_user_input
|
|
21
21
|
from pipelime.piper import PipelimeCommand
|
|
22
22
|
from pipelime.stages import StageInput
|
|
23
|
+
from pipelime.utils.pydantic_compat import iter_fields
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
@dataclass(frozen=True)
|
|
@@ -143,8 +144,8 @@ class TuiApp(App[Mapping]):
|
|
|
143
144
|
"""
|
|
144
145
|
tui_fields = {}
|
|
145
146
|
|
|
146
|
-
for field in self.cmd_cls
|
|
147
|
-
if field.
|
|
147
|
+
for field in iter_fields(self.cmd_cls):
|
|
148
|
+
if field.inner_type is StageInput:
|
|
148
149
|
tui_fields[field.name] = init_stageinput_tui_field(field, cmd_args)
|
|
149
150
|
else:
|
|
150
151
|
tui_fields[field.name] = init_tui_field(field, cmd_args)
|