pipelime-python 2.1.0__py3-none-any.whl → 2.3.0__py3-none-any.whl

pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "2.1.0"
+__version__ = "2.3.0"

pipelime/cli/utils.py

@@ -816,7 +816,9 @@ def get_user_input(text: str) -> t.Any:
     Returns:
         The parsed user input.
     """
-    live = get_console()._live
+    # if there is only one live, pause it to avoid messing the prompt
+    live_stack = get_console()._live_stack
+    live = live_stack[0] if len(live_stack) == 1 else None
     if live is not None:
         live.stop()
 
@@ -878,7 +880,7 @@ class PipelimeUserAppDir:
         for i in reversed(range(1, cls.MAX_CHECKPOINTS)):
             ckpt = base_ckpt.with_name(base_ckpt.name + f"-{i}")
             if ckpt.exists():
-                ckpt.rename(base_ckpt.with_name(base_ckpt.name + f"-{i+1}"))
+                ckpt.rename(base_ckpt.with_name(base_ckpt.name + f"-{i + 1}"))
 
         ckpt = base_ckpt.with_name(base_ckpt.name + "-1")
         ckpt.mkdir(parents=True, exist_ok=False)

pipelime/commands/piper.py

@@ -25,11 +25,13 @@ class WatcherBackend(Enum):
 
     RICH = "rich"
     TQDM = "tqdm"
+    LOGURU = "loguru"
 
     def listener_key(self) -> str:
         return {
             WatcherBackend.RICH: "RICH_TABLE",
             WatcherBackend.TQDM: "TQDM_BARS",
+            WatcherBackend.LOGURU: "LOGURU",
         }[self]
 
 
@@ -254,8 +256,10 @@ class RunCommandBase(GraphPortForwardingCommand):
 
             PipelimeCommand._track_callback = DirectTrackCallback(callback)
 
-            # disable annoying logging
-            logger.disable("pipelime")
+            # disable annoying logging (only if we don't need it for the watcher)
+            if watch != WatcherBackend.LOGURU:
+                logger.disable("pipelime")
+
             exit_stack.callback(logger.enable, "pipelime")
             exit_stack.callback(PipelimeCommand._track_callback.stop_callbacks)
 
@@ -456,7 +460,8 @@ class WatchCommand(PipelimeCommand, title="watch"):
         from pipelime.utils.context_managers import CatchSignals
 
         receiver = ProgressReceiverFactory.get_receiver(
-            self.token, **({"port": self.port} if self.port else {})  # type: ignore
+            self.token,
+            **({"port": self.port} if self.port else {}),  # type: ignore
         )
         if isinstance(self.watcher, WatcherBackend):
             callback = ListenerCallbackFactory.get_callback(

pipelime/piper/progress/listener/callbacks/loguru.py

@@ -0,0 +1,79 @@
+import ast
+import json
+import os
+from typing import ClassVar
+
+from loguru import logger
+
+from pipelime.piper.progress.listener.base import ListenerCallback
+from pipelime.piper.progress.model import ProgressUpdate
+
+
+class LoguruListenerCallback(ListenerCallback):
+    """Listener callback that sends progress updates to loguru.
+
+    Args:
+        show_token: Whether to include the token in the logged data.
+        log_extras: Extra fields to bind to the logger. Can also be set via
+            environment variables with the prefix PIPELIME_PROGRESS_LOG_EXTRA_
+            (e.g., PIPELIME_PROGRESS_LOG_EXTRA_key=value). Environment variables
+            take precedence over this argument.
+        level: Logging level to use. Can also be set via the environment variable
+            PIPELIME_PROGRESS_LOG_LEVEL. Defaults to "DEBUG".
+    """
+
+    LOG_LEVEL_ENV_VAR: ClassVar[str] = "PIPELIME_PROGRESS_LOG_LEVEL"
+    LOG_EXTRA_ENV_VAR_PREFIX: ClassVar[str] = "PIPELIME_PROGRESS_LOG_EXTRA_"
+
+    @classmethod
+    def update_data(cls, data: dict, prog: ProgressUpdate, show_token: bool) -> None:
+        """Update the data dictionary with the progress update"""
+        root = data.setdefault(prog.op_info.token, dict()) if show_token else data
+
+        chunk_data = root.setdefault(prog.op_info.node, dict()).setdefault(
+            str(prog.op_info.chunk), dict()
+        )
+        chunk_data["message"] = prog.op_info.message
+        chunk_data["total"] = prog.op_info.total
+        chunk_data["progress"] = prog.progress
+        chunk_data["finished"] = prog.finished
+
+    def __init__(
+        self,
+        show_token: bool = False,
+        log_extras: dict | None = None,
+        level: str = "DEBUG",
+    ) -> None:
+        super().__init__(show_token)
+
+        self._data: dict = dict()
+
+        # collect log extras from environment variables
+        env_extras = {}
+        for key, value in os.environ.items():
+            if key.startswith(self.LOG_EXTRA_ENV_VAR_PREFIX):
+                extra_key = key[len(self.LOG_EXTRA_ENV_VAR_PREFIX) :].lower()
+                try:
+                    env_extras[extra_key] = ast.literal_eval(value)
+                except (ValueError, SyntaxError):
+                    env_extras[extra_key] = value
+
+        # merge env extras with provided extras (env vars take precedence)
+        log_extras = log_extras or {}
+        self._log_extras = {**log_extras, **env_extras}
+
+        # determine log level (env var takes precedence)
+        self._level = os.getenv(self.LOG_LEVEL_ENV_VAR, level).upper()
+
+    def _log_data(self) -> None:
+        if self._data:
+            logger.bind(**self._log_extras).log(self._level, json.dumps(self._data))
+
+    def on_update(self, prog: ProgressUpdate) -> None:
+        """Called when a progress update is received"""
+        LoguruListenerCallback.update_data(self._data, prog, self.show_token)
+        self._log_data()
+
+    def on_stop(self) -> None:
+        """Called when the listener stops - ensures final progress is sent"""
+        self._log_data()

pipelime/piper/progress/listener/factory.py

@@ -1,13 +1,15 @@
 from typing import Optional
+
 from pipelime.piper.progress.listener.base import ListenerCallback, ProgressReceiver
-from pipelime.piper.progress.listener.receiver.zmq import ZMQProgressReceiver
+from pipelime.piper.progress.listener.callbacks.file import FileListenerCallback
+from pipelime.piper.progress.listener.callbacks.loguru import LoguruListenerCallback
 from pipelime.piper.progress.listener.callbacks.rich_table import (
     RichTableListenerCallback,
 )
 from pipelime.piper.progress.listener.callbacks.tqdm_bars import (
     TqdmBarsListenerCallback,
 )
-from pipelime.piper.progress.listener.callbacks.file import FileListenerCallback
+from pipelime.piper.progress.listener.receiver.zmq import ZMQProgressReceiver
 
 
 class ProgressReceiverFactory:
@@ -35,6 +37,7 @@ class ListenerCallbackFactory:
         "RICH_TABLE": RichTableListenerCallback,
         "TQDM_BARS": TqdmBarsListenerCallback,
         "FILE": FileListenerCallback,
+        "LOGURU": LoguruListenerCallback,
     }
 
     @classmethod

{pipelime_python-2.1.0.dist-info → pipelime_python-2.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: pipelime-python
-Version: 2.1.0
+Version: 2.3.0
 Summary: Data workflows, cli and dataflow automation.
 Project-URL: Source, https://github.com/eyecan-ai/pipelime-python
 Project-URL: Issues, https://github.com/eyecan-ai/pipelime-python/issues
@@ -19,9 +19,10 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
-Requires-Python: <3.13,>=3.10
+Requires-Python: <3.14,>=3.10
 Requires-Dist: albumentations>=1.0.0
 Requires-Dist: astunparse
 Requires-Dist: deepdiff

{pipelime_python-2.1.0.dist-info → pipelime_python-2.3.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 LICENSE,sha256=Ufc-hKELjY574QxuUAosOjRD6RO0x_Y3jtnNbjRK5pc,1525
-pipelime/__init__.py,sha256=XdKxm5F9mco63-Q1cyLZzd3l0CN1F4ZAfgNnW27y11g,115
+pipelime/__init__.py,sha256=bT9rQeSERfVxEwn3AugYS3SczbXoOrEPBPwq57BN5Sw,115
 pipelime/choixe/__init__.py,sha256=sFXmatlOorxsGTrB0q6W8RUA0KNqnC4qiRbDRbRZrBM,52
 pipelime/choixe/xconfig.py,sha256=b3ensxAIXgLNw_mVb6_yNnvkHEEGvuJmIN120ACnQlI,15124
 pipelime/choixe/ast/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -22,7 +22,7 @@ pipelime/cli/main.py,sha256=jvK-IabaiOJBW7GxGV_175AY73V5wrr_gStRWQcD9XU,34629
 pipelime/cli/parser.py,sha256=8zUpfr5Ijed6dzjomGnT-uVC5xNjBgGl3_Gld8lTKB0,8623
 pipelime/cli/pretty_print.py,sha256=X3jld46DFC_Lmrl9Sejs5Tb5Mmv0bLfPPVdHhR-7fsU,15277
 pipelime/cli/subcommands.py,sha256=NIjzapuj_NVcljd1AWGlVt-r-HOj8noM3nohITta4dk,1705
-pipelime/cli/utils.py,sha256=_xF2T3mjtAQGE1tYcCFb4NuTVu9smBOgWRgevXP9Ne8,28595
+pipelime/cli/utils.py,sha256=H6dXwYO489_jx6_YmvBT4B6hKPzjZDUL8cOzfLL9lhw,28738
 pipelime/cli/tui/__init__.py,sha256=9bmQNLmJHeZIQMcqbU99b7OgrdbeEJKAZ142TD6_fM0,89
 pipelime/cli/tui/tui.css,sha256=_W_Q0eDVT7hAbmBm3KsjA0GnDGheXaasnyZIYC9ZNjU,452
 pipelime/cli/tui/tui.py,sha256=79iNTPe4TvJwkZVnnf-Iqk3rT_b2lntqZon7N5Yxy48,10163
@@ -30,7 +30,7 @@ pipelime/cli/tui/utils.py,sha256=JaVAzfbdLVzrst6NzNCN5CVO0_gOREOy2142ysmmxkk,644
 pipelime/commands/__init__.py,sha256=pk6Xr0MW7FDwtqJPOyzHhz6-vYksRzMw79HlhzhRDj4,741
 pipelime/commands/general.py,sha256=xPJYj5rnS4-aBY4B5AJt9RMZ4cP7JqyQeaG8EaQGGSs,37237
 pipelime/commands/interfaces.py,sha256=aE42ysEy7YMwhzPMrTLoebeYKf4N1KSWKYWuUx_nY3Q,34348
-pipelime/commands/piper.py,sha256=luq7tk5Tr2i2k2KQX9h0DznF3c10yVptAY9Rtmne0YI,23771
+pipelime/commands/piper.py,sha256=HkG2dt9UK1dQTq9HNGQ3rwDnabU-t8lBaqf1ED0kPiM,23946
 pipelime/commands/resume.py,sha256=AGcf8UFttxUD5_Ojq7G41NxsKRD_bjXzkqt6rgXURcg,2581
 pipelime/commands/shell.py,sha256=wrW6WGVP_05t28B3CDHQoRb812_wQr8EHOFPemAnXE0,2502
 pipelime/commands/split_ops.py,sha256=hvYYZ4ReoIn2ngkcL-tVaq5qwy1jHRCx0g0rT64lMR0,13669
@@ -69,9 +69,10 @@ pipelime/piper/progress/estimator/factory.py,sha256=153f9SxceXibIsDELkhzrHAXDVO9
 pipelime/piper/progress/estimator/naive.py,sha256=3NGGCD4IQXEORblfs3XEhERRQ8MVgTcwfl8d57ux3Ds,1382
 pipelime/piper/progress/listener/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pipelime/piper/progress/listener/base.py,sha256=OPHOOs-A0NvCtJkGLmK-r5w6TrPtZ9hCBtQ9XSO21yM,3089
-pipelime/piper/progress/listener/factory.py,sha256=PVLV9pbcGA6XDPTBT1hl6lgg3aM1g7MwC2yWcauFsRg,1295
+pipelime/piper/progress/listener/factory.py,sha256=M4PWel_rlbwMpw0B9rWhggNbZ76Rfyb62yIq4E4u01k,1423
 pipelime/piper/progress/listener/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pipelime/piper/progress/listener/callbacks/file.py,sha256=JWRJYoEtsY9mWLdBf1RoMlhTmsUbWv8kD5hOZCVcrCs,2170
+pipelime/piper/progress/listener/callbacks/loguru.py,sha256=Vb8mjK1A2zXDY9o0Juh2vEHuixnpamN7TseMKsv8CMk,3041
 pipelime/piper/progress/listener/callbacks/rich_table.py,sha256=s10TM1arqMz7NTMp9kZeHe-o9HlvdxlUT5xH3_HmJ7M,3442
 pipelime/piper/progress/listener/callbacks/tqdm_bars.py,sha256=3TqKz6g79jaURplmeQg4SkJhEqVSCxK-CsT6_si82Es,1945
 pipelime/piper/progress/listener/receiver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -113,8 +114,8 @@ pipelime/utils/context_managers.py,sha256=C3ubzFVBPNG_wu1jqKc0o1Vy1sxsZ-UzFKry2g
 pipelime/utils/deprecated.py,sha256=rBlOk3cJPZw3jB7QuIY8U3LPWLuovpj8Mfarn65GrqU,1544
 pipelime/utils/inspection.py,sha256=mpfx-wnoDrocpZ5nlh--XtqEGQEEo9WIYIlF18WXXWg,1369
 pipelime/utils/pydantic_types.py,sha256=OGw1bNdWmIRTFkF-fPBHLzAVqaTtrsaS6-mUZ8zgHSg,24955
-pipelime_python-2.1.0.dist-info/METADATA,sha256=fFTCHqhJSlF9tolV9gHA6L3m2-U_R0f1ncUHXV3LkRg,8509
-pipelime_python-2.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pipelime_python-2.1.0.dist-info/entry_points.txt,sha256=hY61hWhOC6q8tq5Qj5oOOEjZo9dtOrztO7IXylYsoTo,61
-pipelime_python-2.1.0.dist-info/licenses/LICENSE,sha256=Ufc-hKELjY574QxuUAosOjRD6RO0x_Y3jtnNbjRK5pc,1525
-pipelime_python-2.1.0.dist-info/RECORD,,
+pipelime_python-2.3.0.dist-info/METADATA,sha256=QVznwKUF2I2NPMRnMYj0JxS0HihUzd7IKitKGAHccJ4,8560
+pipelime_python-2.3.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+pipelime_python-2.3.0.dist-info/entry_points.txt,sha256=hY61hWhOC6q8tq5Qj5oOOEjZo9dtOrztO7IXylYsoTo,61
+pipelime_python-2.3.0.dist-info/licenses/LICENSE,sha256=Ufc-hKELjY574QxuUAosOjRD6RO0x_Y3jtnNbjRK5pc,1525
+pipelime_python-2.3.0.dist-info/RECORD,,

{pipelime_python-2.1.0.dist-info → pipelime_python-2.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.32.0
 Root-Is-Purelib: true
 Tag: py3-none-any