pipelime-python 2.0.0__tar.gz → 2.2.0__tar.gz

{pipelime_python-2.0.0 → pipelime_python-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipelime-python
-Version: 2.0.0
+Version: 2.2.0
 Summary: Data workflows, cli and dataflow automation.
 Project-URL: Source, https://github.com/eyecan-ai/pipelime-python
 Project-URL: Issues, https://github.com/eyecan-ai/pipelime-python/issues

{pipelime_python-2.0.0 → pipelime_python-2.2.0}/pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "2.0.0"
+__version__ = "2.2.0"

{pipelime_python-2.0.0 → pipelime_python-2.2.0}/pipelime/cli/utils.py

@@ -816,7 +816,9 @@ def get_user_input(text: str) -> t.Any:
     Returns:
         The parsed user input.
     """
-    live = get_console()._live
+    # if there is only one live, pause it to avoid messing the prompt
+    live_stack = get_console()._live_stack
+    live = live_stack[0] if len(live_stack) == 1 else None
     if live is not None:
         live.stop()
 
@@ -878,7 +880,7 @@ class PipelimeUserAppDir:
         for i in reversed(range(1, cls.MAX_CHECKPOINTS)):
             ckpt = base_ckpt.with_name(base_ckpt.name + f"-{i}")
             if ckpt.exists():
-                ckpt.rename(base_ckpt.with_name(base_ckpt.name + f"-{i+1}"))
+                ckpt.rename(base_ckpt.with_name(base_ckpt.name + f"-{i + 1}"))
 
         ckpt = base_ckpt.with_name(base_ckpt.name + "-1")
         ckpt.mkdir(parents=True, exist_ok=False)

{pipelime_python-2.0.0 → pipelime_python-2.2.0}/pipelime/commands/interfaces.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
+import json
 import typing as t
 import uuid
 from pathlib import Path
-from urllib.parse import ParseResult
 
 import pydantic.v1 as pyd
+from pydantic.v1.generics import GenericModel
 
 from pipelime.piper import PiperPortType
 from pipelime.utils.pydantic_types import ItemType, SampleValidationInterface, YamlInput
@@ -835,3 +836,128 @@ class ExtendedInterval(Interval):
             raise ValueError(f"Invalid interval: {value}")
 
         return cls(**data)
+
+
+ValueType = t.TypeVar("ValueType")
+
+
+class OutputValueInterface(
+    PydanticFieldNoDefaultMixin,
+    GenericModel,
+    t.Generic[ValueType],
+    extra="forbid",
+    copy_on_model_validation="none",
+    underscore_attrs_are_private=True,
+):
+    """Interface that allows to store a value in a file.
+
+    Examples:
+        How to use it in your command::
+
+        class EasyCommand(PipelimeCommand, title="easy"):
+            output_value: OutputValueInterface[int] = OutputValueInterface.pyd_field()
+
+            def run(self):
+                # this will write the value 42 to the given file
+                self.output_value.set(42)
+    """
+
+    _default_type_description: t.ClassVar[t.Optional[str]] = "An output value."
+    _compact_form: t.ClassVar[t.Optional[str]] = "<file>[,<exists_ok>]"
+    _default_port_type: t.ClassVar[PiperPortType] = PiperPortType.OUTPUT
+    _data: ValueType
+
+    file: Path = pyd.Field(
+        description=(
+            "The output file that will store the value. If the file exists, it will"
+            "be overwritten if `exists_ok` is set to `True`."
+        ),
+    )
+    exists_ok: bool = pyd.Field(
+        False,
+        description=(
+            "If `True`, the output file will be overwritten if it exists. "
+            "If `False`, an error will be raised."
+        ),
+    )
+
+    @pyd.validator("file")
+    def resolve_file(cls, v: Path):
+        # see https://bugs.python.org/issue38671
+        return v.resolve().absolute()
+
+    @classmethod
+    def validate(cls, value):
+        if isinstance(value, OutputValueInterface):
+            return value
+
+        if isinstance(value, (str, bytes, Path)):
+            data = {}
+            raw_data = str(value).split(",")
+            data["file"] = raw_data[0]
+            if len(raw_data) > 1:
+                # NB: the value is left as-is when it is not True nor False
+                # to raise an error on validation
+                data["exists_ok"] = (
+                    True
+                    if raw_data[1].lower() == "true"
+                    else (False if raw_data[1].lower() == "false" else raw_data[1])
+                )
+            value = data
+
+        if isinstance(value, t.Mapping):
+            return OutputValueInterface(**value)
+
+        raise ValueError("Invalid OutputValueInterface definition.")
+
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @pyd.validator("exists_ok", always=True)
+    def _check_file_exists(cls, v: bool, values: t.Mapping[str, t.Any]) -> bool:
+        if "file" not in values:
+            # the "file" validator already failed
+            return v
+
+        if v is True:
+            if values["file"].is_dir():
+                raise ValueError(
+                    f"Cannot overwrite the directory: `{values['file']}`. "
+                    "Please provide a file path."
+                )
+
+        if v is False:
+            if values["file"].is_dir():
+                raise ValueError(
+                    f"Trying to overwrite the directory: `{values['file']}`. "
+                    "Please provide a file path and use `exists_ok=True` to overwrite."
+                )
+
+            if values["file"].exists():
+                raise ValueError(
+                    f"Trying to overwrite an existing file: `{values['file']}`. "
+                    "Please use `exists_ok=True` to overwrite."
+                )
+
+        return v
+
+    def get(self) -> ValueType:
+        """Return the stored value."""
+        return self._data
+
+    def set(self, value: ValueType) -> None:
+        """Writes the value to the output file.
+
+        Args:
+            value: The value to write.
+        """
+        self._data = value
+        with open(self.file, "w") as f:
+            json.dump(self._data, f)
+
+    def __repr__(self) -> str:
+        return self.__piper_repr__()
+
+    def __piper_repr__(self) -> str:
+        return self.file.as_posix()

{pipelime_python-2.0.0 → pipelime_python-2.2.0}/pipelime/commands/piper.py

@@ -25,11 +25,13 @@ class WatcherBackend(Enum):
 
     RICH = "rich"
     TQDM = "tqdm"
+    LOGURU = "loguru"
 
     def listener_key(self) -> str:
         return {
             WatcherBackend.RICH: "RICH_TABLE",
             WatcherBackend.TQDM: "TQDM_BARS",
+            WatcherBackend.LOGURU: "LOGURU",
         }[self]
 
 
@@ -254,8 +256,10 @@ class RunCommandBase(GraphPortForwardingCommand):
 
             PipelimeCommand._track_callback = DirectTrackCallback(callback)
 
-            # disable annoying logging
-            logger.disable("pipelime")
+            # disable annoying logging (only if we don't need it for the watcher)
+            if watch != WatcherBackend.LOGURU:
+                logger.disable("pipelime")
+
             exit_stack.callback(logger.enable, "pipelime")
             exit_stack.callback(PipelimeCommand._track_callback.stop_callbacks)
 
@@ -456,7 +460,8 @@ class WatchCommand(PipelimeCommand, title="watch"):
         from pipelime.utils.context_managers import CatchSignals
 
         receiver = ProgressReceiverFactory.get_receiver(
-            self.token, **({"port": self.port} if self.port else {})  # type: ignore
+            self.token,
+            **({"port": self.port} if self.port else {}),  # type: ignore
         )
         if isinstance(self.watcher, WatcherBackend):
             callback = ListenerCallbackFactory.get_callback(

pipelime_python-2.2.0/pipelime/piper/progress/listener/callbacks/loguru.py

@@ -0,0 +1,79 @@
+import ast
+import json
+import os
+from typing import ClassVar
+
+from loguru import logger
+
+from pipelime.piper.progress.listener.base import ListenerCallback
+from pipelime.piper.progress.model import ProgressUpdate
+
+
+class LoguruListenerCallback(ListenerCallback):
+    """Listener callback that sends progress updates to loguru.
+
+    Args:
+        show_token: Whether to include the token in the logged data.
+        log_extras: Extra fields to bind to the logger. Can also be set via
+            environment variables with the prefix PIPELIME_PROGRESS_LOG_EXTRA_
+            (e.g., PIPELIME_PROGRESS_LOG_EXTRA_key=value). Environment variables
+            take precedence over this argument.
+        level: Logging level to use. Can also be set via the environment variable
+            PIPELIME_PROGRESS_LOG_LEVEL. Defaults to "DEBUG".
+    """
+
+    LOG_LEVEL_ENV_VAR: ClassVar[str] = "PIPELIME_PROGRESS_LOG_LEVEL"
+    LOG_EXTRA_ENV_VAR_PREFIX: ClassVar[str] = "PIPELIME_PROGRESS_LOG_EXTRA_"
+
+    @classmethod
+    def update_data(cls, data: dict, prog: ProgressUpdate, show_token: bool) -> None:
+        """Update the data dictionary with the progress update"""
+        root = data.setdefault(prog.op_info.token, dict()) if show_token else data
+
+        chunk_data = root.setdefault(prog.op_info.node, dict()).setdefault(
+            str(prog.op_info.chunk), dict()
+        )
+        chunk_data["message"] = prog.op_info.message
+        chunk_data["total"] = prog.op_info.total
+        chunk_data["progress"] = prog.progress
+        chunk_data["finished"] = prog.finished
+
+    def __init__(
+        self,
+        show_token: bool = False,
+        log_extras: dict | None = None,
+        level: str = "DEBUG",
+    ) -> None:
+        super().__init__(show_token)
+
+        self._data: dict = dict()
+
+        # collect log extras from environment variables
+        env_extras = {}
+        for key, value in os.environ.items():
+            if key.startswith(self.LOG_EXTRA_ENV_VAR_PREFIX):
+                extra_key = key[len(self.LOG_EXTRA_ENV_VAR_PREFIX) :].lower()
+                try:
+                    env_extras[extra_key] = ast.literal_eval(value)
+                except (ValueError, SyntaxError):
+                    env_extras[extra_key] = value
+
+        # merge env extras with provided extras (env vars take precedence)
+        log_extras = log_extras or {}
+        self._log_extras = {**log_extras, **env_extras}
+
+        # determine log level (env var takes precedence)
+        self._level = os.getenv(self.LOG_LEVEL_ENV_VAR, level).upper()
+
+    def _log_data(self) -> None:
+        if self._data:
+            logger.bind(**self._log_extras).log(self._level, json.dumps(self._data))
+
+    def on_update(self, prog: ProgressUpdate) -> None:
+        """Called when a progress update is received"""
+        LoguruListenerCallback.update_data(self._data, prog, self.show_token)
+        self._log_data()
+
+    def on_stop(self) -> None:
+        """Called when the listener stops - ensures final progress is sent"""
+        self._log_data()

{pipelime_python-2.0.0 → pipelime_python-2.2.0}/pipelime/piper/progress/listener/factory.py

@@ -1,13 +1,15 @@
 from typing import Optional
+
 from pipelime.piper.progress.listener.base import ListenerCallback, ProgressReceiver
-from pipelime.piper.progress.listener.receiver.zmq import ZMQProgressReceiver
+from pipelime.piper.progress.listener.callbacks.file import FileListenerCallback
+from pipelime.piper.progress.listener.callbacks.loguru import LoguruListenerCallback
 from pipelime.piper.progress.listener.callbacks.rich_table import (
     RichTableListenerCallback,
 )
 from pipelime.piper.progress.listener.callbacks.tqdm_bars import (
     TqdmBarsListenerCallback,
 )
-from pipelime.piper.progress.listener.callbacks.file import FileListenerCallback
+from pipelime.piper.progress.listener.receiver.zmq import ZMQProgressReceiver
 
 
 class ProgressReceiverFactory:
@@ -35,6 +37,7 @@ class ListenerCallbackFactory:
         "RICH_TABLE": RichTableListenerCallback,
         "TQDM_BARS": TqdmBarsListenerCallback,
         "FILE": FileListenerCallback,
+        "LOGURU": LoguruListenerCallback,
     }
 
     @classmethod