pipelime-python 1.9.0__tar.gz → 2.0.0__tar.gz

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: pipelime-python
-Version: 1.9.0
+Version: 2.0.0
 Summary: Data workflows, cli and dataflow automation.
 Project-URL: Source, https://github.com/eyecan-ai/pipelime-python
 Project-URL: Issues, https://github.com/eyecan-ai/pipelime-python/issues
@@ -16,23 +16,22 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Natural Language :: English
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
-Requires-Python: >=3.8
+Requires-Python: <3.13,>=3.10
 Requires-Dist: albumentations>=1.0.0
 Requires-Dist: astunparse
-Requires-Dist: billiard
 Requires-Dist: deepdiff
 Requires-Dist: dictquery
 Requires-Dist: filelock
-Requires-Dist: imageio>=2.17.0
+Requires-Dist: imageio[ffmpeg,pyav]>=2.17.0
 Requires-Dist: loguru
-Requires-Dist: minio
+Requires-Dist: networkx
 Requires-Dist: numpy
-Requires-Dist: pydantic<2,>=1.10.8
+Requires-Dist: pydantic<3,>=1.10.17
 Requires-Dist: pydash
 Requires-Dist: python-box
 Requires-Dist: pyyaml
@@ -64,6 +63,7 @@ Provides-Extra: tests
 Requires-Dist: pytest; extra == 'tests'
 Requires-Dist: pytest-asyncio; extra == 'tests'
 Requires-Dist: pytest-cov; extra == 'tests'
+Requires-Dist: tox; extra == 'tests'
 Description-Content-Type: text/markdown
 
 
@@ -79,7 +79,7 @@ Description-Content-Type: text/markdown
 Welcome to **pipelime**, a swiss army knife for data processing!
 
 `pipelime` is a full-fledge **framework** for **data science**: read your datasets,
-manipulate them, write back to disk or upload to a remote data lake.
+manipulate them and write back to disk.
 Then build up your **dataflow** with Piper and manage the configuration with Choixe.
 Finally, **embed** your custom commands into the `pipelime` workspace, to act both as dataflow nodes and advanced command line interface.
 

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/README.md

@@ -11,7 +11,7 @@
 Welcome to **pipelime**, a swiss army knife for data processing!
 
 `pipelime` is a full-fledge **framework** for **data science**: read your datasets,
-manipulate them, write back to disk or upload to a remote data lake.
+manipulate them and write back to disk.
 Then build up your **dataflow** with Piper and manage the configuration with Choixe.
 Finally, **embed** your custom commands into the `pipelime` workspace, to act both as dataflow nodes and advanced command line interface.
 

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "1.9.0"
+__version__ = "2.0.0"

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/choixe/ast/nodes.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from dataclasses import field
 from typing import Any, Dict, Optional, Sequence, Tuple, Union
 
-# from dataclasses import dataclass
-from pydantic.dataclasses import dataclass
+from pydantic.v1.dataclasses import dataclass
 
 
 class NodeVisitor:  # pragma: no cover
@@ -336,7 +336,7 @@ class RandNode(Node):
 
     args: Sequence[HashNode] = tuple()
     n: Optional[Node] = None
-    pdf: Optional[Node] = None
+    pdf: Optional[Node] = field(default=None, compare=False)
 
     def __init__(self, *args: HashNode, **data) -> None:
         super().__init__()

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/choixe/utils/io.py

@@ -111,9 +111,9 @@ class PipelimeTmp:
 
         user_dirs: Dict[str, List[str]] = {}
         for d in iglob(str(PipelimeTmp.base_dir() / (PipelimeTmp.prefix() + "*"))):
-            if os.path.isdir(d):
+            if os.path.isdir(d):  # pragma: no branch
                 names = os.path.basename(d).split("-")
-                if len(names) >= 3:
+                if len(names) >= 3:  # pragma: no branch
                     user_dirs.setdefault(names[2], []).append(d)
         return user_dirs
 
@@ -135,7 +135,8 @@ class PipelimeTemporaryDirectory:
         import shutil
 
         def onerror(func, path, exc_info):
-            if issubclass(exc_info[0], PermissionError):
+            # this code comes from TemporaryDirectory
+            if issubclass(exc_info[0], PermissionError):  # pragma: no cover
 
                 def resetperms(path):
                     try:

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/choixe/visitors/decoder.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from typing import Any
 
 import numpy as np
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 from pipelime.choixe.ast.nodes import LiteralNode, Node
 from pipelime.choixe.visitors.unparser import Unparser

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/cli/main.py

@@ -6,7 +6,7 @@ import typing as t
 from pathlib import Path
 
 import typer
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 from pipelime.choixe import XConfig
 from pipelime.choixe.visitors.processor_ui import ProcessorUi
@@ -35,6 +35,7 @@ class PlCliOptions(BaseModel):
     output_ctx: t.Optional[Path]
     command_outputs: t.Optional[Path]
     verbose: int
+    data_cache: t.Optional[str]
     dry_run: bool
     no_ui: bool
     command: str
@@ -57,7 +58,57 @@ class PlCliOptions(BaseModel):
         return data
 
 
-def _complete_yaml(incomplete: str):
+def _set_logger_level(verbose: int):
+    """Set the logger level if not forced by the user."""
+    import os
+    import sys
+
+    from loguru import logger
+
+    if "LOGURU_LEVEL" not in os.environ:
+        if verbose == 0:
+            level = "WARNING"
+        elif verbose == 1:
+            level = "INFO"
+        elif verbose == 2:
+            level = "DEBUG"
+        else:
+            level = "TRACE"
+        logger.remove()
+        logger.add(sys.stderr, level=level)
+
+        # set the level in the environment so that it is inherited by subprocesses
+        os.environ["LOGURU_LEVEL"] = level
+
+
+def _set_item_data_cache(data_cache: t.Optional[str]):
+    from loguru import logger
+
+    from pipelime.items import enable_item_data_cache
+    from pipelime.utils.pydantic_types import ItemType
+
+    item_list = (data_cache or "").replace(" ", "").split(",")
+    item_list = [i for i in item_list if i]
+
+    if not item_list:
+        return
+
+    if "*" in item_list:
+        logger.debug("Enabling data cache for all items.")
+        enable_item_data_cache()
+    else:
+        item_cls_list = []
+        for name in item_list:
+            try:
+                item_cls_list.append(ItemType.create(name).value)
+            except Exception:
+                logger.error(f"Item `{name}` not found.")
+        logger.debug(f"Enabling data cache for items: {item_cls_list}")
+        if item_cls_list:
+            enable_item_data_cache(*item_cls_list)
+
+
+def _complete_yaml(incomplete: str):  # pragma: no cover
     for v in Path(".").glob(f"{incomplete}*.yaml"):
         yield str(v)
 
@@ -399,6 +450,13 @@ def pl_main(
         resolve_path=True,
         help="The checkpoint folder. If it already exists, it must be empty.",
     ),
+    data_cache: t.Optional[str] = typer.Option(
+        None,
+        help=(
+            "Comma-separated list of pipelime items for which to enable data cache. "
+            "Use the special `*` value to enable data cache for any item."
+        ),
+    ),
     verbose: int = typer.Option(
         0,
         "--verbose",
@@ -468,6 +526,8 @@ def pl_main(
     TRUE boolean values. Use `false` or `true` to explicitly set a boolean
     and `none`/`null`/`nul` to enforce `None`.
     """
+    _set_logger_level(verbose)
+
     PipelimeSymbolsHelper.set_extra_modules(extra_modules)
 
     if command_args is None:
@@ -549,6 +609,7 @@ def pl_main(
             output_ctx=output_ctx,
             command_outputs=command_outputs,
             verbose=verbose,
+            data_cache=data_cache,
             dry_run=dry_run,
             no_ui=no_ui,
             command=command,
@@ -593,6 +654,7 @@ def run_with_checkpoint(
     from pipelime.choixe import XConfig
     from pipelime.cli.pretty_print import print_error, print_info, print_warning
 
+    _set_item_data_cache(cli_opts.data_cache)
     PipelimeSymbolsHelper.set_extra_modules(cli_opts.extra_modules)
 
     if cli_opts.pipelime_tmp:
@@ -780,7 +842,9 @@ def run_with_checkpoint(
 
         cfg_size = len(effective_configs)
         pls = "s" if cfg_size != 1 else ""
-        print_info(f"🎉 Configuration successfully processed ({cfg_size} variant{pls}).")
+        print_info(
+            f"🎉 Configuration successfully processed ({cfg_size} variant{pls})."
+        )
 
         if cli_opts.verbose > 2:
             print_info("\nFinal effective configurations:")
@@ -854,7 +918,7 @@ def run_command(
 
     import time
 
-    from pydantic.error_wrappers import ValidationError
+    from pydantic.v1.error_wrappers import ValidationError
 
     from pipelime.choixe.utils.io import PipelimeTmp, dump
     from pipelime.cli.pretty_print import print_command_outputs, print_info

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/cli/pretty_print.py

@@ -1,14 +1,13 @@
 import inspect
 import typing as t
 
-from pydantic import BaseModel
-from rich import box
-from rich.markup import escape
-from rich import get_console
+from pydantic.v1 import BaseModel
+from rich import box, get_console
 from rich import print as rprint
-from rich.pretty import Pretty
-from rich.table import Table, Column
+from rich.markup import escape
 from rich.panel import Panel
+from rich.pretty import Pretty
+from rich.table import Column, Table
 
 if t.TYPE_CHECKING:
     from pipelime.cli.utils import ActionInfo
@@ -421,12 +420,13 @@ def _is_model(type_):
 
 def _human_readable_type(field_outer_type):
     from enum import Enum
-    from pydantic.typing import (
-        typing_base,  # noqa: F401  # type: ignore
+
+    from pydantic.v1.typing import (
         WithArgsTypes,
-        is_union,
-        get_origin,
         get_args,
+        get_origin,
+        is_union,
+        typing_base,  # noqa: F401  # type: ignore
     )
 
     v = field_outer_type

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/cli/tui/tui.css

@@ -19,7 +19,7 @@
 }
 
 .description {
-    height: 0;
+    height: auto;
     opacity: 0.5;
 }
 

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/cli/tui/tui.py

@@ -30,18 +30,16 @@ class Constants:
     SUB_FIELD_MARGIN = (0, 0, 0, 4)
     TUI_KEY_CONFIRM = Keys.ControlN
     TUI_KEY_SAVE = Keys.ControlS
-    TUI_KEY_ABORT = Keys.ControlC
+    TUI_KEY_ABORT = Keys.ControlT
     TUI_KEY_TOGGLE_DESCRIPTIONS = Keys.ControlJ
     SAVE_KEY_CONFIRM = Keys.ControlS
     SAVE_KEY_CANCEL = Keys.Escape
-    SAVE_KEY_ABORT = Keys.ControlC
 
 
 class SaveScreen(ModalScreen):
     BINDINGS = [
         (Constants.SAVE_KEY_CONFIRM, "confirm", "Confirm"),
         (Constants.SAVE_KEY_CANCEL, "cancel", "Cancel"),
-        (Constants.SAVE_KEY_ABORT, "ctrl_c", "Abort"),
     ]
     HELP = "CTRL+S to confirm, ESC to cancel"
 
@@ -55,11 +53,14 @@ class SaveScreen(ModalScreen):
         self.config = config
 
     def compose(self) -> ComposeResult:
+        """Compose the save screen."""
+        error_label = Label("", classes="error-label")
+        error_label.display = False
         yield Container(
             Label("Save path", classes="field-label"),
             Input(""),
-            Label(SaveScreen.HELP),
-            Label("", classes="error-label"),
+            Label(str(SaveScreen.HELP)),
+            error_label,
             id="dialog",
         )
 
@@ -85,17 +86,16 @@ class SaveScreen(ModalScreen):
         except Exception as e:
             error_label = self.query_one(".error-label")
             error_label = cast(Label, error_label)
-            error_label.styles.height = "auto"
-            error_label.update(str(e))
+            error_label.display = True
+            error = str(e)
+            # escape the square brackets to avoid rich syntax
+            error = error.replace("[", r"\[")  # noqa: W605
+            error_label.update(error)
 
     def action_cancel(self) -> None:
         """Cancel the save."""
         self.app.pop_screen()
 
-    def action_ctrl_c(self) -> None:
-        """Propagate the KeyboardInterrupt exception."""
-        raise KeyboardInterrupt
-
 
 class TuiApp(App[Mapping]):
     """A Textual app to handle Pipelime configurations."""
@@ -104,7 +104,7 @@ class TuiApp(App[Mapping]):
     BINDINGS = [
         (Constants.TUI_KEY_CONFIRM, "confirm", "Confirm"),
         (Constants.TUI_KEY_SAVE, "save", "Save to file"),
-        (Constants.TUI_KEY_ABORT, "ctrl_c", "Abort"),
+        (Constants.TUI_KEY_ABORT, "abort", "Abort"),
         (
             Constants.TUI_KEY_TOGGLE_DESCRIPTIONS,
             "toggle_descriptions",
@@ -165,7 +165,9 @@ class TuiApp(App[Mapping]):
             labels.append(Label(title, classes="title-label"))
         if description:
             description = TuiApp.preprocess_string(description)
-            labels.append(Label(description, classes="title-label description"))
+            label = Label(description, classes="title-label description")
+            label.display = True if self.show_descriptions else False
+            labels.append(label)
         return labels
 
     def create_simple_field(self, field: TuiField) -> List[Widget]:
@@ -186,7 +188,9 @@ class TuiApp(App[Mapping]):
         description = field.description
         descr = f"({field.type_}) {description}" if description else f"({field.type_})"
         description = TuiApp.preprocess_string(descr)
-        widgets.append(Label(description, classes="description"))
+        label = Label(description, classes="description")
+        label.display = True if self.show_descriptions else False
+        widgets.append(label)
 
         inp = Input(value=field.value, placeholder=field.hint)
         widgets.append(inp)
@@ -275,8 +279,8 @@ class TuiApp(App[Mapping]):
         save_screen = SaveScreen(self.collect_cmd_args())
         self.push_screen(save_screen)
 
-    def action_ctrl_c(self) -> None:
-        """Propagate the KeyboardInterrupt exception."""
+    def action_abort(self) -> None:
+        """Raise KeyboardInterrupt exception."""
         raise KeyboardInterrupt
 
     def action_toggle_descriptions(self) -> None:
@@ -284,7 +288,7 @@ class TuiApp(App[Mapping]):
         self.show_descriptions = not self.show_descriptions
         query = self.query(".description")
         for widget in query:
-            widget.styles.height = "auto" if self.show_descriptions else 0
+            widget.display = True if self.show_descriptions else False
 
         query = self.query(Input)
         for widget in query:
@@ -315,7 +319,7 @@ class TuiApp(App[Mapping]):
                 replace_whitespace=False,
                 tabsize=4,
             )
-            sub = sub.replace("[", "\[")  # noqa: W605
+            sub = sub.replace("[", r"\[")  # noqa: W605
             preprocessed += sub + "\n"
 
         preprocessed = preprocessed[:-1]

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/cli/tui/utils.py

@@ -1,8 +1,8 @@
 from enum import Enum
 from typing import List, Mapping, Tuple, Type, cast
 
-from pydantic import BaseModel
-from pydantic.fields import ModelField
+from pydantic.v1 import BaseModel
+from pydantic.v1.fields import ModelField
 
 from pipelime.cli.utils import PipelimeSymbolsHelper
 from pipelime.piper import PipelimeCommand

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/cli/utils.py

@@ -1,24 +1,24 @@
 import json
+import os
+import shutil
 import typing as t
 from ast import literal_eval
 from json import JSONDecodeError
 from pathlib import Path
 from types import ModuleType
-import shutil
 
 import yaml
 from loguru import logger
-from pydantic import BaseModel, ValidationError
+from pydantic.v1 import BaseModel, ValidationError
 from rich import get_console
 from rich.prompt import Prompt
 from yaml.error import YAMLError
-import os
 
 if t.TYPE_CHECKING:
     from pipelime.piper import T_DAG_NODE, PipelimeCommand
     from pipelime.piper.checkpoint import CheckpointNamespace
-    from pipelime.stages import SampleStage
     from pipelime.sequences import SamplesSequence
+    from pipelime.stages import SampleStage
 
 
 class ActionInfo(BaseModel):
@@ -108,7 +108,7 @@ class PipelimeSymbolsHelper:
 
     @classmethod
     def _symbol_name(cls, symbol):
-        from pydantic import BaseModel
+        from pydantic.v1 import BaseModel
 
         return (
             symbol.__config__.title
@@ -175,9 +175,9 @@ class PipelimeSymbolsHelper:
     def import_everything(cls):
         import pipelime.choixe.utils.imports as pl_imports
         import pipelime.sequences as pl_seq
+        from pipelime.commands.piper import DagBaseCommand
         from pipelime.piper import PipelimeCommand
         from pipelime.stages import SampleStage
-        from pipelime.commands.piper import DagBaseCommand
 
         if not cls.is_cache_valid():
             for module_name in cls.std_modules + cls.extra_modules:
@@ -631,7 +631,7 @@ def pl_print(
     """
     import inspect
 
-    from pydantic import BaseModel
+    from pydantic.v1 import BaseModel
     from rich import print as rprint
 
     from pipelime.cli.pretty_print import print_model_info

{pipelime_python-1.9.0 → pipelime_python-2.0.0}/pipelime/commands/__init__.py

@@ -1,29 +1,27 @@
 from pipelime.commands.general import (
-    TimeItCommand,
-    StageTimingCommand,
-    PipeCommand,
     CloneCommand,
     ConcatCommand,
-    ZipCommand,
-    AddRemoteCommand,
-    RemoveRemoteCommand,
-    ValidateCommand,
+    CopySharedItemsCommand,
+    FilterCommand,
+    FilterDuplicatesCommand,
     MapCommand,
     MapIfCommand,
-    SortCommand,
-    FilterCommand,
-    SliceCommand,
+    PipeCommand,
     SetMetadataCommand,
-    FilterDuplicatesCommand,
-    CopySharedItemsCommand,
+    SliceCommand,
+    SortCommand,
+    StageTimingCommand,
+    TimeItCommand,
+    ValidateCommand,
+    ZipCommand,
 )
+from pipelime.commands.piper import DrawCommand, RunCommand, WatchCommand
+from pipelime.commands.resume import ResumeCommand
 from pipelime.commands.shell import ShellCommand
 from pipelime.commands.split_ops import (
     SplitByQueryCommand,
     SplitByValueCommand,
     SplitCommand,
 )
-from pipelime.commands.toy_dataset import ToyDatasetCommand
-from pipelime.commands.piper import RunCommand, DrawCommand, WatchCommand
 from pipelime.commands.tempman import TempCommand
-from pipelime.commands.resume import ResumeCommand
+from pipelime.commands.toy_dataset import ToyDatasetCommand