pipelime-python 2.1.0__tar.gz → 2.2.0__tar.gz

{pipelime_python-2.1.0 → pipelime_python-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipelime-python
-Version: 2.1.0
+Version: 2.2.0
 Summary: Data workflows, cli and dataflow automation.
 Project-URL: Source, https://github.com/eyecan-ai/pipelime-python
 Project-URL: Issues, https://github.com/eyecan-ai/pipelime-python/issues

{pipelime_python-2.1.0 → pipelime_python-2.2.0}/pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "2.1.0"
+__version__ = "2.2.0"

{pipelime_python-2.1.0 → pipelime_python-2.2.0}/pipelime/cli/utils.py

@@ -816,7 +816,9 @@ def get_user_input(text: str) -> t.Any:
     Returns:
         The parsed user input.
     """
-    live = get_console()._live
+    # if there is only one live, pause it to avoid messing the prompt
+    live_stack = get_console()._live_stack
+    live = live_stack[0] if len(live_stack) == 1 else None
     if live is not None:
         live.stop()
 
@@ -878,7 +880,7 @@ class PipelimeUserAppDir:
         for i in reversed(range(1, cls.MAX_CHECKPOINTS)):
             ckpt = base_ckpt.with_name(base_ckpt.name + f"-{i}")
             if ckpt.exists():
-                ckpt.rename(base_ckpt.with_name(base_ckpt.name + f"-{i+1}"))
+                ckpt.rename(base_ckpt.with_name(base_ckpt.name + f"-{i + 1}"))
 
         ckpt = base_ckpt.with_name(base_ckpt.name + "-1")
         ckpt.mkdir(parents=True, exist_ok=False)

{pipelime_python-2.1.0 → pipelime_python-2.2.0}/pipelime/commands/piper.py

@@ -25,11 +25,13 @@ class WatcherBackend(Enum):
 
     RICH = "rich"
     TQDM = "tqdm"
+    LOGURU = "loguru"
 
     def listener_key(self) -> str:
         return {
             WatcherBackend.RICH: "RICH_TABLE",
             WatcherBackend.TQDM: "TQDM_BARS",
+            WatcherBackend.LOGURU: "LOGURU",
         }[self]
 
 
@@ -254,8 +256,10 @@ class RunCommandBase(GraphPortForwardingCommand):
 
             PipelimeCommand._track_callback = DirectTrackCallback(callback)
 
-            # disable annoying logging
-            logger.disable("pipelime")
+            # disable annoying logging (only if we don't need it for the watcher)
+            if watch != WatcherBackend.LOGURU:
+                logger.disable("pipelime")
+
             exit_stack.callback(logger.enable, "pipelime")
             exit_stack.callback(PipelimeCommand._track_callback.stop_callbacks)
 
@@ -456,7 +460,8 @@ class WatchCommand(PipelimeCommand, title="watch"):
         from pipelime.utils.context_managers import CatchSignals
 
         receiver = ProgressReceiverFactory.get_receiver(
-            self.token, **({"port": self.port} if self.port else {})  # type: ignore
+            self.token,
+            **({"port": self.port} if self.port else {}),  # type: ignore
         )
         if isinstance(self.watcher, WatcherBackend):
             callback = ListenerCallbackFactory.get_callback(

pipelime_python-2.2.0/pipelime/piper/progress/listener/callbacks/loguru.py

@@ -0,0 +1,79 @@
+import ast
+import json
+import os
+from typing import ClassVar
+
+from loguru import logger
+
+from pipelime.piper.progress.listener.base import ListenerCallback
+from pipelime.piper.progress.model import ProgressUpdate
+
+
+class LoguruListenerCallback(ListenerCallback):
+    """Listener callback that sends progress updates to loguru.
+
+    Args:
+        show_token: Whether to include the token in the logged data.
+        log_extras: Extra fields to bind to the logger. Can also be set via
+            environment variables with the prefix PIPELIME_PROGRESS_LOG_EXTRA_
+            (e.g., PIPELIME_PROGRESS_LOG_EXTRA_key=value). Environment variables
+            take precedence over this argument.
+        level: Logging level to use. Can also be set via the environment variable
+            PIPELIME_PROGRESS_LOG_LEVEL. Defaults to "DEBUG".
+    """
+
+    LOG_LEVEL_ENV_VAR: ClassVar[str] = "PIPELIME_PROGRESS_LOG_LEVEL"
+    LOG_EXTRA_ENV_VAR_PREFIX: ClassVar[str] = "PIPELIME_PROGRESS_LOG_EXTRA_"
+
+    @classmethod
+    def update_data(cls, data: dict, prog: ProgressUpdate, show_token: bool) -> None:
+        """Update the data dictionary with the progress update"""
+        root = data.setdefault(prog.op_info.token, dict()) if show_token else data
+
+        chunk_data = root.setdefault(prog.op_info.node, dict()).setdefault(
+            str(prog.op_info.chunk), dict()
+        )
+        chunk_data["message"] = prog.op_info.message
+        chunk_data["total"] = prog.op_info.total
+        chunk_data["progress"] = prog.progress
+        chunk_data["finished"] = prog.finished
+
+    def __init__(
+        self,
+        show_token: bool = False,
+        log_extras: dict | None = None,
+        level: str = "DEBUG",
+    ) -> None:
+        super().__init__(show_token)
+
+        self._data: dict = dict()
+
+        # collect log extras from environment variables
+        env_extras = {}
+        for key, value in os.environ.items():
+            if key.startswith(self.LOG_EXTRA_ENV_VAR_PREFIX):
+                extra_key = key[len(self.LOG_EXTRA_ENV_VAR_PREFIX) :].lower()
+                try:
+                    env_extras[extra_key] = ast.literal_eval(value)
+                except (ValueError, SyntaxError):
+                    env_extras[extra_key] = value
+
+        # merge env extras with provided extras (env vars take precedence)
+        log_extras = log_extras or {}
+        self._log_extras = {**log_extras, **env_extras}
+
+        # determine log level (env var takes precedence)
+        self._level = os.getenv(self.LOG_LEVEL_ENV_VAR, level).upper()
+
+    def _log_data(self) -> None:
+        if self._data:
+            logger.bind(**self._log_extras).log(self._level, json.dumps(self._data))
+
+    def on_update(self, prog: ProgressUpdate) -> None:
+        """Called when a progress update is received"""
+        LoguruListenerCallback.update_data(self._data, prog, self.show_token)
+        self._log_data()
+
+    def on_stop(self) -> None:
+        """Called when the listener stops - ensures final progress is sent"""
+        self._log_data()

{pipelime_python-2.1.0 → pipelime_python-2.2.0}/pipelime/piper/progress/listener/factory.py

@@ -1,13 +1,15 @@
 from typing import Optional
+
 from pipelime.piper.progress.listener.base import ListenerCallback, ProgressReceiver
-from pipelime.piper.progress.listener.receiver.zmq import ZMQProgressReceiver
+from pipelime.piper.progress.listener.callbacks.file import FileListenerCallback
+from pipelime.piper.progress.listener.callbacks.loguru import LoguruListenerCallback
 from pipelime.piper.progress.listener.callbacks.rich_table import (
     RichTableListenerCallback,
 )
 from pipelime.piper.progress.listener.callbacks.tqdm_bars import (
     TqdmBarsListenerCallback,
 )
-from pipelime.piper.progress.listener.callbacks.file import FileListenerCallback
+from pipelime.piper.progress.listener.receiver.zmq import ZMQProgressReceiver
 
 
 class ProgressReceiverFactory:
@@ -35,6 +37,7 @@ class ListenerCallbackFactory:
         "RICH_TABLE": RichTableListenerCallback,
         "TQDM_BARS": TqdmBarsListenerCallback,
         "FILE": FileListenerCallback,
+        "LOGURU": LoguruListenerCallback,
     }
 
     @classmethod