pipelime-python 1.9.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "1.9.0"
+__version__ = "2.0.0"

pipelime/choixe/ast/nodes.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from dataclasses import field
 from typing import Any, Dict, Optional, Sequence, Tuple, Union
 
-# from dataclasses import dataclass
-from pydantic.dataclasses import dataclass
+from pydantic.v1.dataclasses import dataclass
 
 
 class NodeVisitor:  # pragma: no cover
@@ -336,7 +336,7 @@ class RandNode(Node):
 
     args: Sequence[HashNode] = tuple()
     n: Optional[Node] = None
-    pdf: Optional[Node] = None
+    pdf: Optional[Node] = field(default=None, compare=False)
 
     def __init__(self, *args: HashNode, **data) -> None:
         super().__init__()

pipelime/choixe/utils/io.py

@@ -111,9 +111,9 @@ class PipelimeTmp:
 
         user_dirs: Dict[str, List[str]] = {}
         for d in iglob(str(PipelimeTmp.base_dir() / (PipelimeTmp.prefix() + "*"))):
-            if os.path.isdir(d):
+            if os.path.isdir(d):  # pragma: no branch
                 names = os.path.basename(d).split("-")
-                if len(names) >= 3:
+                if len(names) >= 3:  # pragma: no branch
                     user_dirs.setdefault(names[2], []).append(d)
         return user_dirs
 
@@ -135,7 +135,8 @@ class PipelimeTemporaryDirectory:
         import shutil
 
         def onerror(func, path, exc_info):
-            if issubclass(exc_info[0], PermissionError):
+            # this code comes from TemporaryDirectory
+            if issubclass(exc_info[0], PermissionError):  # pragma: no cover
 
                 def resetperms(path):
                     try:

pipelime/choixe/visitors/decoder.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from typing import Any
 
 import numpy as np
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 from pipelime.choixe.ast.nodes import LiteralNode, Node
 from pipelime.choixe.visitors.unparser import Unparser

pipelime/cli/main.py

@@ -6,7 +6,7 @@ import typing as t
 from pathlib import Path
 
 import typer
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 from pipelime.choixe import XConfig
 from pipelime.choixe.visitors.processor_ui import ProcessorUi
@@ -35,6 +35,7 @@ class PlCliOptions(BaseModel):
     output_ctx: t.Optional[Path]
     command_outputs: t.Optional[Path]
     verbose: int
+    data_cache: t.Optional[str]
     dry_run: bool
     no_ui: bool
     command: str
@@ -57,7 +58,57 @@ class PlCliOptions(BaseModel):
         return data
 
 
-def _complete_yaml(incomplete: str):
+def _set_logger_level(verbose: int):
+    """Set the logger level if not forced by the user."""
+    import os
+    import sys
+
+    from loguru import logger
+
+    if "LOGURU_LEVEL" not in os.environ:
+        if verbose == 0:
+            level = "WARNING"
+        elif verbose == 1:
+            level = "INFO"
+        elif verbose == 2:
+            level = "DEBUG"
+        else:
+            level = "TRACE"
+        logger.remove()
+        logger.add(sys.stderr, level=level)
+
+        # set the level in the environment so that it is inherited by subprocesses
+        os.environ["LOGURU_LEVEL"] = level
+
+
+def _set_item_data_cache(data_cache: t.Optional[str]):
+    from loguru import logger
+
+    from pipelime.items import enable_item_data_cache
+    from pipelime.utils.pydantic_types import ItemType
+
+    item_list = (data_cache or "").replace(" ", "").split(",")
+    item_list = [i for i in item_list if i]
+
+    if not item_list:
+        return
+
+    if "*" in item_list:
+        logger.debug("Enabling data cache for all items.")
+        enable_item_data_cache()
+    else:
+        item_cls_list = []
+        for name in item_list:
+            try:
+                item_cls_list.append(ItemType.create(name).value)
+            except Exception:
+                logger.error(f"Item `{name}` not found.")
+        logger.debug(f"Enabling data cache for items: {item_cls_list}")
+        if item_cls_list:
+            enable_item_data_cache(*item_cls_list)
+
+
+def _complete_yaml(incomplete: str):  # pragma: no cover
     for v in Path(".").glob(f"{incomplete}*.yaml"):
         yield str(v)
 
@@ -399,6 +450,13 @@ def pl_main(
         resolve_path=True,
         help="The checkpoint folder. If it already exists, it must be empty.",
     ),
+    data_cache: t.Optional[str] = typer.Option(
+        None,
+        help=(
+            "Comma-separated list of pipelime items for which to enable data cache. "
+            "Use the special `*` value to enable data cache for any item."
+        ),
+    ),
     verbose: int = typer.Option(
         0,
         "--verbose",
@@ -468,6 +526,8 @@ def pl_main(
     TRUE boolean values. Use `false` or `true` to explicitly set a boolean
     and `none`/`null`/`nul` to enforce `None`.
     """
+    _set_logger_level(verbose)
+
     PipelimeSymbolsHelper.set_extra_modules(extra_modules)
 
     if command_args is None:
@@ -549,6 +609,7 @@ def pl_main(
             output_ctx=output_ctx,
             command_outputs=command_outputs,
             verbose=verbose,
+            data_cache=data_cache,
             dry_run=dry_run,
             no_ui=no_ui,
             command=command,
@@ -593,6 +654,7 @@ def run_with_checkpoint(
     from pipelime.choixe import XConfig
     from pipelime.cli.pretty_print import print_error, print_info, print_warning
 
+    _set_item_data_cache(cli_opts.data_cache)
     PipelimeSymbolsHelper.set_extra_modules(cli_opts.extra_modules)
 
     if cli_opts.pipelime_tmp:
@@ -780,7 +842,9 @@ def run_with_checkpoint(
 
         cfg_size = len(effective_configs)
         pls = "s" if cfg_size != 1 else ""
-        print_info(f"🎉 Configuration successfully processed ({cfg_size} variant{pls}).")
+        print_info(
+            f"🎉 Configuration successfully processed ({cfg_size} variant{pls})."
+        )
 
         if cli_opts.verbose > 2:
             print_info("\nFinal effective configurations:")
@@ -854,7 +918,7 @@ def run_command(
 
     import time
 
-    from pydantic.error_wrappers import ValidationError
+    from pydantic.v1.error_wrappers import ValidationError
 
     from pipelime.choixe.utils.io import PipelimeTmp, dump
     from pipelime.cli.pretty_print import print_command_outputs, print_info

pipelime/cli/pretty_print.py

@@ -1,14 +1,13 @@
 import inspect
 import typing as t
 
-from pydantic import BaseModel
-from rich import box
-from rich.markup import escape
-from rich import get_console
+from pydantic.v1 import BaseModel
+from rich import box, get_console
 from rich import print as rprint
-from rich.pretty import Pretty
-from rich.table import Table, Column
+from rich.markup import escape
 from rich.panel import Panel
+from rich.pretty import Pretty
+from rich.table import Column, Table
 
 if t.TYPE_CHECKING:
     from pipelime.cli.utils import ActionInfo
@@ -421,12 +420,13 @@ def _is_model(type_):
 
 def _human_readable_type(field_outer_type):
     from enum import Enum
-    from pydantic.typing import (
-        typing_base,  # noqa: F401  # type: ignore
+
+    from pydantic.v1.typing import (
         WithArgsTypes,
-        is_union,
-        get_origin,
         get_args,
+        get_origin,
+        is_union,
+        typing_base,  # noqa: F401  # type: ignore
     )
 
     v = field_outer_type

pipelime/cli/tui/tui.css

@@ -19,7 +19,7 @@
 }
 
 .description {
-    height: 0;
+    height: auto;
     opacity: 0.5;
 }
 

pipelime/cli/tui/tui.py

@@ -30,18 +30,16 @@ class Constants:
     SUB_FIELD_MARGIN = (0, 0, 0, 4)
     TUI_KEY_CONFIRM = Keys.ControlN
     TUI_KEY_SAVE = Keys.ControlS
-    TUI_KEY_ABORT = Keys.ControlC
+    TUI_KEY_ABORT = Keys.ControlT
     TUI_KEY_TOGGLE_DESCRIPTIONS = Keys.ControlJ
     SAVE_KEY_CONFIRM = Keys.ControlS
     SAVE_KEY_CANCEL = Keys.Escape
-    SAVE_KEY_ABORT = Keys.ControlC
 
 
 class SaveScreen(ModalScreen):
     BINDINGS = [
         (Constants.SAVE_KEY_CONFIRM, "confirm", "Confirm"),
         (Constants.SAVE_KEY_CANCEL, "cancel", "Cancel"),
-        (Constants.SAVE_KEY_ABORT, "ctrl_c", "Abort"),
     ]
     HELP = "CTRL+S to confirm, ESC to cancel"
 
@@ -55,11 +53,14 @@ class SaveScreen(ModalScreen):
         self.config = config
 
     def compose(self) -> ComposeResult:
+        """Compose the save screen."""
+        error_label = Label("", classes="error-label")
+        error_label.display = False
         yield Container(
             Label("Save path", classes="field-label"),
             Input(""),
-            Label(SaveScreen.HELP),
-            Label("", classes="error-label"),
+            Label(str(SaveScreen.HELP)),
+            error_label,
             id="dialog",
         )
 
@@ -85,17 +86,16 @@ class SaveScreen(ModalScreen):
         except Exception as e:
             error_label = self.query_one(".error-label")
             error_label = cast(Label, error_label)
-            error_label.styles.height = "auto"
-            error_label.update(str(e))
+            error_label.display = True
+            error = str(e)
+            # escape the square brackets to avoid rich syntax
+            error = error.replace("[", r"\[")  # noqa: W605
+            error_label.update(error)
 
     def action_cancel(self) -> None:
         """Cancel the save."""
         self.app.pop_screen()
 
-    def action_ctrl_c(self) -> None:
-        """Propagate the KeyboardInterrupt exception."""
-        raise KeyboardInterrupt
-
 
 class TuiApp(App[Mapping]):
     """A Textual app to handle Pipelime configurations."""
@@ -104,7 +104,7 @@ class TuiApp(App[Mapping]):
     BINDINGS = [
         (Constants.TUI_KEY_CONFIRM, "confirm", "Confirm"),
         (Constants.TUI_KEY_SAVE, "save", "Save to file"),
-        (Constants.TUI_KEY_ABORT, "ctrl_c", "Abort"),
+        (Constants.TUI_KEY_ABORT, "abort", "Abort"),
         (
             Constants.TUI_KEY_TOGGLE_DESCRIPTIONS,
             "toggle_descriptions",
@@ -165,7 +165,9 @@ class TuiApp(App[Mapping]):
             labels.append(Label(title, classes="title-label"))
         if description:
             description = TuiApp.preprocess_string(description)
-            labels.append(Label(description, classes="title-label description"))
+            label = Label(description, classes="title-label description")
+            label.display = True if self.show_descriptions else False
+            labels.append(label)
         return labels
 
     def create_simple_field(self, field: TuiField) -> List[Widget]:
@@ -186,7 +188,9 @@ class TuiApp(App[Mapping]):
         description = field.description
         descr = f"({field.type_}) {description}" if description else f"({field.type_})"
         description = TuiApp.preprocess_string(descr)
-        widgets.append(Label(description, classes="description"))
+        label = Label(description, classes="description")
+        label.display = True if self.show_descriptions else False
+        widgets.append(label)
 
         inp = Input(value=field.value, placeholder=field.hint)
         widgets.append(inp)
@@ -275,8 +279,8 @@ class TuiApp(App[Mapping]):
         save_screen = SaveScreen(self.collect_cmd_args())
         self.push_screen(save_screen)
 
-    def action_ctrl_c(self) -> None:
-        """Propagate the KeyboardInterrupt exception."""
+    def action_abort(self) -> None:
+        """Raise KeyboardInterrupt exception."""
         raise KeyboardInterrupt
 
     def action_toggle_descriptions(self) -> None:
@@ -284,7 +288,7 @@ class TuiApp(App[Mapping]):
         self.show_descriptions = not self.show_descriptions
         query = self.query(".description")
         for widget in query:
-            widget.styles.height = "auto" if self.show_descriptions else 0
+            widget.display = True if self.show_descriptions else False
 
         query = self.query(Input)
         for widget in query:
@@ -315,7 +319,7 @@ class TuiApp(App[Mapping]):
                 replace_whitespace=False,
                 tabsize=4,
             )
-            sub = sub.replace("[", "\[")  # noqa: W605
+            sub = sub.replace("[", r"\[")  # noqa: W605
             preprocessed += sub + "\n"
 
         preprocessed = preprocessed[:-1]

pipelime/cli/tui/utils.py

@@ -1,8 +1,8 @@
 from enum import Enum
 from typing import List, Mapping, Tuple, Type, cast
 
-from pydantic import BaseModel
-from pydantic.fields import ModelField
+from pydantic.v1 import BaseModel
+from pydantic.v1.fields import ModelField
 
 from pipelime.cli.utils import PipelimeSymbolsHelper
 from pipelime.piper import PipelimeCommand

pipelime/cli/utils.py

@@ -1,24 +1,24 @@
 import json
+import os
+import shutil
 import typing as t
 from ast import literal_eval
 from json import JSONDecodeError
 from pathlib import Path
 from types import ModuleType
-import shutil
 
 import yaml
 from loguru import logger
-from pydantic import BaseModel, ValidationError
+from pydantic.v1 import BaseModel, ValidationError
 from rich import get_console
 from rich.prompt import Prompt
 from yaml.error import YAMLError
-import os
 
 if t.TYPE_CHECKING:
     from pipelime.piper import T_DAG_NODE, PipelimeCommand
     from pipelime.piper.checkpoint import CheckpointNamespace
-    from pipelime.stages import SampleStage
     from pipelime.sequences import SamplesSequence
+    from pipelime.stages import SampleStage
 
 
 class ActionInfo(BaseModel):
@@ -108,7 +108,7 @@ class PipelimeSymbolsHelper:
 
     @classmethod
     def _symbol_name(cls, symbol):
-        from pydantic import BaseModel
+        from pydantic.v1 import BaseModel
 
         return (
             symbol.__config__.title
@@ -175,9 +175,9 @@ class PipelimeSymbolsHelper:
     def import_everything(cls):
         import pipelime.choixe.utils.imports as pl_imports
         import pipelime.sequences as pl_seq
+        from pipelime.commands.piper import DagBaseCommand
         from pipelime.piper import PipelimeCommand
         from pipelime.stages import SampleStage
-        from pipelime.commands.piper import DagBaseCommand
 
         if not cls.is_cache_valid():
             for module_name in cls.std_modules + cls.extra_modules:
@@ -631,7 +631,7 @@ def pl_print(
     """
     import inspect
 
-    from pydantic import BaseModel
+    from pydantic.v1 import BaseModel
     from rich import print as rprint
 
     from pipelime.cli.pretty_print import print_model_info

pipelime/commands/__init__.py

@@ -1,29 +1,27 @@
 from pipelime.commands.general import (
-    TimeItCommand,
-    StageTimingCommand,
-    PipeCommand,
     CloneCommand,
     ConcatCommand,
-    ZipCommand,
-    AddRemoteCommand,
-    RemoveRemoteCommand,
-    ValidateCommand,
+    CopySharedItemsCommand,
+    FilterCommand,
+    FilterDuplicatesCommand,
     MapCommand,
     MapIfCommand,
-    SortCommand,
-    FilterCommand,
-    SliceCommand,
+    PipeCommand,
     SetMetadataCommand,
-    FilterDuplicatesCommand,
-    CopySharedItemsCommand,
+    SliceCommand,
+    SortCommand,
+    StageTimingCommand,
+    TimeItCommand,
+    ValidateCommand,
+    ZipCommand,
 )
+from pipelime.commands.piper import DrawCommand, RunCommand, WatchCommand
+from pipelime.commands.resume import ResumeCommand
 from pipelime.commands.shell import ShellCommand
 from pipelime.commands.split_ops import (
     SplitByQueryCommand,
     SplitByValueCommand,
     SplitCommand,
 )
-from pipelime.commands.toy_dataset import ToyDatasetCommand
-from pipelime.commands.piper import RunCommand, DrawCommand, WatchCommand
 from pipelime.commands.tempman import TempCommand
-from pipelime.commands.resume import ResumeCommand
+from pipelime.commands.toy_dataset import ToyDatasetCommand

pipelime/commands/general.py

@@ -1,6 +1,6 @@
 import typing as t
 
-import pydantic as pyd
+import pydantic.v1 as pyd
 
 import pipelime.commands.interfaces as pl_interfaces
 import pipelime.utils.pydantic_types as pl_types
@@ -37,22 +37,22 @@ class OutputStageTime(pyd.BaseModel):
 class TimeItCommand(PipelimeCommand, title="timeit"):
     """Measures the average time to get a sample from a sequence."""
 
-    input: t.Optional[
-        pl_interfaces.InputDatasetInterface
-    ] = pl_interfaces.InputDatasetInterface.pyd_field(
-        alias="i",
-        is_required=False,
-        description=(
-            "The input dataset. If None, the first operation "
-            "must be a sequence generator."
-        ),
-        piper_port=PiperPortType.INPUT,
+    input: t.Optional[pl_interfaces.InputDatasetInterface] = (
+        pl_interfaces.InputDatasetInterface.pyd_field(
+            alias="i",
+            is_required=False,
+            description=(
+                "The input dataset. If None, the first operation "
+                "must be a sequence generator."
+            ),
+            piper_port=PiperPortType.INPUT,
+        )
     )
 
-    output: t.Optional[
-        pl_interfaces.OutputDatasetInterface
-    ] = pl_interfaces.OutputDatasetInterface.pyd_field(
-        alias="o", is_required=False, piper_port=PiperPortType.OUTPUT
+    output: t.Optional[pl_interfaces.OutputDatasetInterface] = (
+        pl_interfaces.OutputDatasetInterface.pyd_field(
+            alias="o", is_required=False, piper_port=PiperPortType.OUTPUT
+        )
     )
 
     operations: t.Optional[pl_types.YamlInput] = pyd.Field(
@@ -287,16 +287,16 @@ class PipeCommand(PipelimeCommand, title="pipe"):
         ),
     )
 
-    input: t.Optional[
-        pl_interfaces.InputDatasetInterface
-    ] = pl_interfaces.InputDatasetInterface.pyd_field(
-        alias="i",
-        is_required=False,
-        description=(
-            "The input dataset. If None, the first operation "
-            "must be a sequence generator."
-        ),
-        piper_port=PiperPortType.INPUT,
+    input: t.Optional[pl_interfaces.InputDatasetInterface] = (
+        pl_interfaces.InputDatasetInterface.pyd_field(
+            alias="i",
+            is_required=False,
+            description=(
+                "The input dataset. If None, the first operation "
+                "must be a sequence generator."
+            ),
+            piper_port=PiperPortType.INPUT,
+        )
     )
 
     output: pl_interfaces.OutputDatasetInterface = (
@@ -332,11 +332,7 @@ class PipeCommand(PipelimeCommand, title="pipe"):
 
 
 class CloneCommand(PipelimeCommand, title="clone"):
-    """Clone a dataset.
-
-    You can use this command to create a local copy of a dataset
-    hosted on a remote data lake by disabling the `REMOTE_FILE` serialization option.
-    """
+    """Clone a dataset."""
 
     input: pl_interfaces.InputDatasetInterface = (
         pl_interfaces.InputDatasetInterface.pyd_field(
@@ -470,118 +466,6 @@ class ZipCommand(PipelimeCommand, title="zip"):
         )
 
 
-class RemoteCommandBase(PipelimeCommand):
-    input: pl_interfaces.InputDatasetInterface = (
-        pl_interfaces.InputDatasetInterface.pyd_field(
-            alias="i", piper_port=PiperPortType.INPUT
-        )
-    )
-
-    remotes: t.Union[
-        pl_interfaces.RemoteInterface, t.Sequence[pl_interfaces.RemoteInterface]
-    ] = pl_interfaces.RemoteInterface.pyd_field(alias="r")
-
-    keys: t.Union[str, t.Sequence[str]] = pyd.Field(
-        default_factory=list,
-        alias="k",
-        description="Affected keys. Leave empty to take all the keys.",
-    )
-
-    start: int = pyd.Field(
-        0,
-        description=(
-            "The first sample (included), defaults to the first element. "
-            "Can be negative, in which case it counts from the end."
-        ),
-    )
-    stop: t.Optional[int] = pyd.Field(
-        None,
-        description=(
-            "The last sample (excluded), defaults to the whole sequence."
-            "Can be negative, in which case it counts from the end."
-        ),
-    )
-    step: int = pyd.Field(
-        1, description="The slice step, defaults to 1. Can be negative."
-    )
-
-    output: pl_interfaces.OutputDatasetInterface = (
-        pl_interfaces.OutputDatasetInterface.pyd_field(
-            alias="o", piper_port=PiperPortType.OUTPUT
-        )
-    )
-
-    grabber: pl_interfaces.GrabberInterface = pl_interfaces.GrabberInterface.pyd_field(
-        alias="g"
-    )
-
-    @pyd.validator("remotes", "keys")
-    def validate_remotes(cls, v):
-        return (
-            v if not isinstance(v, (str, bytes)) and isinstance(v, t.Sequence) else [v]
-        )
-
-    def _run_remote_op(self, stage, message):
-        from pipelime.sequences.pipes.mapping import MappingConditionIndexRange
-
-        seq = self.input.create_reader().map_if(
-            stage=stage,
-            condition=MappingConditionIndexRange(
-                start=self.start,
-                stop=self.stop,
-                step=self.step,
-            ),
-        )
-
-        self.grabber.grab_all(
-            self.output.append_writer(seq),
-            grab_context_manager=self.output.serialization_cm(),
-            keep_order=False,
-            parent_cmd=self,
-            track_message=message,
-        )
-
-
-class AddRemoteCommand(RemoteCommandBase, title="remote-add"):
-    """Upload samples to one or more remotes.
-
-    Slicing and key options filter the samples to upload,
-    but the whole dataset is always written out.
-    """
-
-    def run(self):
-        from pipelime.stages import StageUploadToRemote
-
-        self._run_remote_op(
-            StageUploadToRemote(
-                remotes=[r.get_url() for r in self.remotes],  # type: ignore
-                keys_to_upload=self.keys,
-            ),
-            "Uploading data",
-        )
-
-
-class RemoveRemoteCommand(RemoteCommandBase, title="remote-remove"):
-    """Remove one or more remote from a dataset.
-
-    Slicing and key options filter the samples,
-    but the whole dataset is always written out.
-
-    NB: data is not removed from the remote data lake.
-    """
-
-    def run(self):
-        from pipelime.stages import StageForgetSource
-
-        remotes = [r.get_url() for r in self.remotes]  # type: ignore
-        remove_all = remotes if not self.keys else []
-        remove_by_key = {k: remotes for k in self.keys}
-
-        self._run_remote_op(
-            StageForgetSource(*remove_all, **remove_by_key), "Removing remote sources"
-        )
-
-
 class ValidateCommand(PipelimeCommand, title="validate"):
     """Outputs a minimal schema which will validate the given input."""