pipelime-python 2.3.0__tar.gz → 3.0.1__tar.gz

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelime-python
-Version: 2.3.0
+Version: 3.0.1
 Summary: Data workflows, cli and dataflow automation.
 Project-URL: Source, https://github.com/eyecan-ai/pipelime-python
 Project-URL: Issues, https://github.com/eyecan-ai/pipelime-python/issues
@@ -32,7 +32,8 @@ Requires-Dist: imageio[ffmpeg,pyav]>=2.17.0
 Requires-Dist: loguru
 Requires-Dist: networkx
 Requires-Dist: numpy
-Requires-Dist: pydantic<3,>=1.10.17
+Requires-Dist: pydantic-extra-types
+Requires-Dist: pydantic<3,>=2.10
 Requires-Dist: pydash
 Requires-Dist: python-box
 Requires-Dist: pyyaml
@@ -64,6 +65,7 @@ Provides-Extra: tests
 Requires-Dist: pytest; extra == 'tests'
 Requires-Dist: pytest-asyncio; extra == 'tests'
 Requires-Dist: pytest-cov; extra == 'tests'
+Requires-Dist: pytest-xdist; extra == 'tests'
 Requires-Dist: tox; extra == 'tests'
 Description-Content-Type: text/markdown
 
@@ -101,6 +103,8 @@ Install `pipelime` using pip:
 pip install pipelime-python
 ```
 
+pipelime ≥ 3.0 requires pydantic v2 — see [`docs/migration/pydantic_v2.md`](docs/migration/pydantic_v2.md) if you upgrade from 2.x.
+
 To be able to *draw* the dataflow graphs, you need the `draw` variant:
 
 ```

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/README.md

@@ -32,6 +32,8 @@ Install `pipelime` using pip:
 pip install pipelime-python
 ```
 
+pipelime ≥ 3.0 requires pydantic v2 — see [`docs/migration/pydantic_v2.md`](docs/migration/pydantic_v2.md) if you upgrade from 2.x.
+
 To be able to *draw* the dataflow graphs, you need the `draw` variant:
 
 ```

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "2.3.0"
+__version__ = "3.0.1"

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/choixe/ast/nodes.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
+import functools
+import types
+import typing
 from abc import ABC, abstractmethod
-from dataclasses import field
+from dataclasses import dataclass, field, fields
 from typing import Any, Dict, Optional, Sequence, Tuple, Union
 
-from pydantic.v1.dataclasses import dataclass
-
 
 class NodeVisitor:  # pragma: no cover
     """Base class for all Choixe visitors. A visitor can be seen as a special recursive
@@ -87,10 +88,47 @@ class NodeVisitor:  # pragma: no cover
         return self._ignore(node)
 
 
+@functools.lru_cache(maxsize=None)
+def _node_typed_fields(cls: type) -> Tuple[Tuple[str, Tuple[type, ...], bool], ...]:
+    """For each dataclass field of `cls` whose resolved annotation is a `Node`
+    subclass, `Optional[<Node subclass>]`, or a `Union` of `Node` subclasses,
+    returns `(field_name, allowed_node_types, allow_none)`. Container-typed
+    fields (e.g. `Dict[...]`, `Sequence[...]`) and `Any` are left out. Cached
+    per class (outside any instance, since `HashNode.__hash__` hashes
+    `self.__dict__.values()`)."""
+    hints = typing.get_type_hints(cls)
+    specs = []
+    for f in fields(cls):
+        hint = hints.get(f.name)
+        is_union = typing.get_origin(hint) is Union or isinstance(hint, types.UnionType)
+        candidates = typing.get_args(hint) if is_union else (hint,)
+        node_types = tuple(
+            c for c in candidates if isinstance(c, type) and issubclass(c, Node)
+        )
+        if node_types:
+            specs.append((f.name, node_types, type(None) in candidates))
+    return tuple(specs)
+
+
 @dataclass
 class Node(ABC):
     """A generic element of the Choixe AST, all nodes must implement this interface."""
 
+    def __post_init__(self) -> None:
+        for name, node_types, allow_none in _node_typed_fields(type(self)):
+            value = getattr(self, name, None)
+            if value is None and allow_none:
+                continue
+            if isinstance(value, node_types):
+                continue
+            expected = " or ".join(t.__name__ for t in node_types)
+            if allow_none:
+                expected += " or None"
+            raise TypeError(
+                f"{type(self).__name__}.{name} expected {expected}, "
+                f"got {type(value).__name__}"
+            )
+
     @abstractmethod
     def accept(self, visitor: NodeVisitor) -> Any:
         """Accepts an incoming visitor. This method should simply call the respective

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/choixe/utils/common.py

@@ -1,5 +1,39 @@
+import re
 import typing as t
 
+# pydash's string path delimiter (the same in pydash 8.0 and 8.1): an unescaped `.`,
+# or a `[<int>]` list index (captured, so that it is kept as a path part)
+_PYDASH_PATH_DELIM = re.compile(r"(?<!\\)(?:\\\\)*\.|(\[-?\d+\])")
+
+
+def pydash_path(key_path: t.Any) -> t.Any:
+    r"""A key path for pydash (``get``, ``set_``, ``has``, ``to_path``) addressing the
+    same value on every pydash version.
+
+    pydash 8.1 reads the empty key of a leading, trailing or doubled ``.`` in a string
+    path (``".a"`` is ``["", "a"]``, ``"."`` is ``["", ""]``), where earlier versions
+    dropped it (``["a"]``, ``[]``: the whole object). pipelime keeps the pre-8.1 meaning:
+    the empty keys of a dotted or indexed string path are dropped here, and the other
+    parts, escapes included, are joined back into a canonical path (``"a.b[0].c"``) that
+    every pydash version splits the same way. A path left with a single key is returned
+    unescaped (pydash takes a plain key as is), with no key as ``[]``. Any other path (a
+    plain key without ``.`` or ``[``, a key list, an int) is returned unchanged.
+    """
+    if not (isinstance(key_path, str) and ("." in key_path or "[" in key_path)):
+        return key_path
+    # `re.split` alternates the text between two delimiters (a key, "" if empty) and
+    # the captured group (a `[<int>]` list index, None for a `.`)
+    parts = _PYDASH_PATH_DELIM.split(key_path)
+    path = ""
+    for i, part in enumerate(parts):
+        if part:
+            path += part if i % 2 or not path else "." + part
+    if not path:
+        return []
+    if "." in path or "[" in path:
+        return path
+    return path.replace("\\\\", "\\")  # pydash's unescape; no `\.` is left here
+
 
 class _RefItem:
     def __init__(self, parent, idx):
@@ -73,7 +107,7 @@ def deep_set_(  # noqa: C901
     from pydash.utilities import to_path
 
     if key_path is not None:  # pragma: no branch
-        key_path_tokens = to_path(key_path)
+        key_path_tokens = to_path(pydash_path(key_path))
         parent_node = _RefItem(obj, None)
         for tk in key_path_tokens:
             if isinstance(tk, int):

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/choixe/visitors/decoder.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from typing import Any
 
 import numpy as np
-from pydantic.v1 import BaseModel
+from pydantic import BaseModel
 
 from pipelime.choixe.ast.nodes import LiteralNode, Node
 from pipelime.choixe.visitors.unparser import Unparser
@@ -20,7 +20,10 @@ class Decoder(Unparser):
             return data.item()
         elif isinstance(data, BaseModel):
             symbol = f"{data.__module__}.{data.__class__.__qualname__}"
-            return {"$model": symbol, "$args": json.loads(data.json(by_alias=True))}
+            return {
+                "$model": symbol,
+                "$args": json.loads(data.model_dump_json(by_alias=True)),
+            }
         elif isinstance(data, Path):
             return str(data)
         else:

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/choixe/visitors/inspector.py

@@ -10,6 +10,7 @@ import pydash as py_
 
 import pipelime.choixe.ast.nodes as ast
 from pipelime.choixe.ast.parser import parse
+from pipelime.choixe.utils.common import pydash_path
 from pipelime.choixe.utils.io import load
 
 
@@ -87,11 +88,11 @@ class Inspector(ast.NodeVisitor):
 
             id_ = node.identifier.data  # type: ignore
             default = None if node.default is None else unparse(node.default)
-            variables = py_.set_({}, id_, default)
+            variables = py_.set_({}, pydash_path(id_), default)
 
             help_strings = {}
             if node.help is not None:
-                help_strings = py_.set_({}, id_, node.help.data)
+                help_strings = py_.set_({}, pydash_path(id_), node.help.data)
 
             environ = {}
             if node.env is not None and node.env.data:  # type: ignore
@@ -150,7 +151,7 @@ class Inspector(ast.NodeVisitor):
         if node.identifier is not None:  # pragma: no branch
             self._named_for_loops[node.identifier.data] = node.iterable.data
         iterable_insp = (
-            Inspection(variables=py_.set_({}, node.iterable.data, None))
+            Inspection(variables=py_.set_({}, pydash_path(node.iterable.data), None))
             if isinstance(node.iterable.data, str)
             else Inspection(processed=True)
         )
@@ -160,7 +161,9 @@ class Inspector(ast.NodeVisitor):
     def visit_switch(self, node: ast.SwitchNode) -> Inspection:
         insp = node.value.accept(self)
         if insp.processed:  # pragma: no branch
-            insp = Inspection(variables=py_.set_({}, node.value.data, None))  # type: ignore
+            insp = Inspection(
+                variables=py_.set_({}, pydash_path(node.value.data), None)  # type: ignore
+            )
         cases_insp = sum(
             [x.accept(self) + y.accept(self) for x, y in node.cases],
             start=Inspection(processed=True),
@@ -186,7 +189,7 @@ class Inspector(ast.NodeVisitor):
                 if key:
                     iterable_name = self._named_for_loops[loop_id]
                     full_path = f"{iterable_name}.{key}"
-                    py_.set_(variables, full_path, None)
+                    py_.set_(variables, pydash_path(full_path), None)
             insp = insp + sub_insp + Inspection(variables=variables)
         return insp
 

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/choixe/visitors/processor.py

@@ -11,6 +11,7 @@ import pydash as py_
 
 import pipelime.choixe.ast.nodes as ast
 from pipelime.choixe.ast.parser import parse
+from pipelime.choixe.utils.common import pydash_path
 from pipelime.choixe.utils.imports import import_symbol
 from pipelime.choixe.utils.io import PipelimeTmp, load
 from pipelime.choixe.utils.rand import rand
@@ -148,8 +149,8 @@ class Processor(ast.NodeVisitor):
             var_value = None
             found = False
 
-            if py_.has(self._context, id_):
-                var_value = py_.get(self._context, id_)
+            if py_.has(self._context, pydash_path(id_)):
+                var_value = py_.get(self._context, pydash_path(id_))
                 found = True
             if not found and env:
                 value = os.getenv(id_)
@@ -214,14 +215,24 @@ class Processor(ast.NodeVisitor):
         symbol_branches = node.symbol.accept(self)
         args_branches = node.args.accept(self)
         branches = self._branches(symbol_branches, args_branches)
-        return [import_symbol(s, cwd=self._cwd).parse_obj(a) for s, a in branches]
+        models = []
+        for s, a in branches:
+            model_cls = import_symbol(s, cwd=self._cwd)
+            if not hasattr(model_cls, "model_validate"):
+                raise TypeError(
+                    f"`$model` needs a pydantic v2 model, got {model_cls!r}: "
+                    "pipelime 3 no longer supports pydantic.v1 models "
+                    "(see docs/migration/pydantic_v2.md)"
+                )
+            models.append(model_cls.model_validate(a))
+        return models
 
     def visit_for(self, node: ast.ForNode) -> List[Any]:
         if isinstance(node.iterable.data, str):
-            if not py_.has(self._context, node.iterable.data):
+            if not py_.has(self._context, pydash_path(node.iterable.data)):
                 iterable = self._handle_missing_for(node.iterable.data)
             else:
-                iterable = py_.get(self._context, node.iterable.data)
+                iterable = py_.get(self._context, pydash_path(node.iterable.data))
         else:
             iterable = node.iterable.data
 
@@ -278,10 +289,10 @@ class Processor(ast.NodeVisitor):
         for branch in all_branches:
             varname = branch[0]
 
-            if not py_.has(self._context, varname):
+            if not py_.has(self._context, pydash_path(varname)):
                 value = self._handle_missing_switch(varname)
             else:
-                value = py_.get(self._context, varname)
+                value = py_.get(self._context, pydash_path(varname))
 
             # Match the value to the correct case
             for i in range(len(node.cases)):
@@ -321,7 +332,7 @@ class Processor(ast.NodeVisitor):
         items = []
         for branch in branches:
             loop_id, _, key = str(branch).partition(sep)  # type: ignore
-            item = py_.get(self._loop_data[loop_id].item, f"{sep}{key}")
+            item = py_.get(self._loop_data[loop_id].item, pydash_path(f"{sep}{key}"))
             items.append(item)
         return items
 

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/choixe/xconfig.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from copy import deepcopy
 from pathlib import Path
 from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
 
@@ -9,7 +10,7 @@ from schema import Schema
 
 from pipelime.choixe.ast.nodes import Node
 from pipelime.choixe.ast.parser import parse
-from pipelime.choixe.utils.common import deep_set_
+from pipelime.choixe.utils.common import deep_set_, pydash_path
 from pipelime.choixe.utils.io import dump, load
 from pipelime.choixe.visitors import Inspection, decode, inspect, process, walk
 
@@ -38,6 +39,11 @@ class XConfig(Box):
                 validation. Defaults to None.
         """
 
+        # Initialize the box config. Nested containers must be plain `Box` objects:
+        # `XConfig.__init__` does not accept box options, and recent python-box
+        # versions would otherwise default `box_class` to the subclass itself.
+        super().__init__(box_class=Box)
+
         # options
         self._cwd = cwd
         self._schema = None
@@ -93,6 +99,15 @@ class XConfig(Box):
         """
         return XConfig(data=self.decode(), cwd=self.get_cwd(), schema=self.get_schema())
 
+    def __deepcopy__(self, memo: Optional[Dict[int, Any]] = None) -> XConfig:
+        # `Box.__deepcopy__` would build the copy passing box options to
+        # `XConfig.__init__` (or return a plain `Box`, on older python-box versions)
+        return XConfig(
+            data=deepcopy(self.to_dict(), memo),
+            cwd=self.get_cwd(),
+            schema=self.get_schema(),
+        )
+
     def validate(self, replace: bool = True):
         """Validate internal schema, if any
 
@@ -139,7 +154,7 @@ class XConfig(Box):
         Returns:
             Any: The value at the specified path.
         """
-        return py_.get(self, full_key, default=default)
+        return py_.get(self, pydash_path(full_key), default=default)
 
     def deep_set(
         self,
@@ -159,7 +174,7 @@ class XConfig(Box):
                 Defaults to False.
         """
 
-        if not only_valid_keys or py_.has(self, full_key):
+        if not only_valid_keys or py_.has(self, pydash_path(full_key)):
             deep_set_(self, key_path=full_key, value=value, append=append_values)
 
     def deep_update(

{pipelime_python-2.3.0 → pipelime_python-3.0.1}/pipelime/cli/main.py

@@ -6,7 +6,6 @@ import typing as t
 from pathlib import Path
 
 import typer
-from pydantic.v1 import BaseModel
 
 from pipelime.choixe import XConfig
 from pipelime.choixe.visitors.processor_ui import ProcessorUi
@@ -18,12 +17,15 @@ from pipelime.cli.utils import (
     print_command_op_stage_info,
     print_commands_ops_stages_list,
 )
+from pipelime.utils.pydantic_compat import PipelimeModel
 
 if t.TYPE_CHECKING:
     from pipelime.piper.checkpoint import Checkpoint
 
 
-class PlCliOptions(BaseModel):
+class PlCliOptions(PipelimeModel):
+    # a `PipelimeModel` for the v1 Optional rule: the options saved in a checkpoint
+    # may lack an `Optional` key
     _namespace: t.ClassVar[str] = "__plmain"
 
     config: t.List[Path]
@@ -43,7 +45,7 @@ class PlCliOptions(BaseModel):
     pipelime_tmp: t.Optional[str]
 
     def purged_dict(self):
-        return self._purge(self.dict())
+        return self._purge(self.model_dump())
 
     def _purge(self, data):
         if isinstance(data, Path):
@@ -918,14 +920,19 @@ def run_command(
 
     import time
 
-    from pydantic.v1.error_wrappers import ValidationError
+    from pydantic import ValidationError
+    from rich.markup import escape
 
     from pipelime.choixe.utils.io import PipelimeTmp, dump
-    from pipelime.cli.pretty_print import print_command_outputs, print_info
+    from pipelime.cli.pretty_print import (
+        print_command_outputs,
+        print_error,
+        print_info,
+    )
     from pipelime.cli.tui import TuiApp, is_tui_needed
     from pipelime.cli.utils import (
+        format_validation_error,
         get_pipelime_command_cls,
-        show_field_alias_valerr,
         time_to_str,
     )
     from pipelime.commands import TempCommand
@@ -972,12 +979,13 @@ def run_command(
             # so let's show the tui again if it was needed in the first place
 
     except ValidationError as e:
-        show_field_alias_valerr(e)
+        # plain text, not Rich markup: `[type=...]` and bracketed values must survive
+        print_error(escape(format_validation_error(e, cmd_cls)))
         raise e
 
     if verbose > 0:
         print_info(f"\nCreated command `{command}`:")
-        print_info(cmd_obj.dict(), pretty=True)
+        print_info(cmd_obj.model_dump(), pretty=True)
 
     if dry_run or verbose > 0:
         print_info(f"\nRunning `{command}`...")