pipelime-python 1.9.0__tar.gz → 1.9.1__tar.gz

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pipelime-python
-Version: 1.9.0
+Version: 1.9.1
 Summary: Data workflows, cli and dataflow automation.
 Project-URL: Source, https://github.com/eyecan-ai/pipelime-python
 Project-URL: Issues, https://github.com/eyecan-ai/pipelime-python/issues

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "1.9.0"
+__version__ = "1.9.1"

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/choixe/utils/io.py

@@ -111,9 +111,9 @@ class PipelimeTmp:
 
         user_dirs: Dict[str, List[str]] = {}
         for d in iglob(str(PipelimeTmp.base_dir() / (PipelimeTmp.prefix() + "*"))):
-            if os.path.isdir(d):
+            if os.path.isdir(d):  # pragma: no branch
                 names = os.path.basename(d).split("-")
-                if len(names) >= 3:
+                if len(names) >= 3:  # pragma: no branch
                     user_dirs.setdefault(names[2], []).append(d)
         return user_dirs
 
@@ -135,7 +135,8 @@ class PipelimeTemporaryDirectory:
         import shutil
 
         def onerror(func, path, exc_info):
-            if issubclass(exc_info[0], PermissionError):
+            # this code comes from TemporaryDirectory
+            if issubclass(exc_info[0], PermissionError):  # pragma: no cover
 
                 def resetperms(path):
                     try:

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/cli/main.py

@@ -57,7 +57,30 @@ class PlCliOptions(BaseModel):
         return data
 
 
-def _complete_yaml(incomplete: str):
+def _set_logger_level(verbose: int):
+    """Set the logger level if not forced by the user."""
+    import os
+    import sys
+
+    from loguru import logger
+
+    if "LOGURU_LEVEL" not in os.environ:
+        if verbose == 0:
+            level = "WARNING"
+        elif verbose == 1:
+            level = "INFO"
+        elif verbose == 2:
+            level = "DEBUG"
+        else:
+            level = "TRACE"
+        logger.remove()
+        logger.add(sys.stderr, level=level)
+
+        # set the level in the environment so that it is inherited by subprocesses
+        os.environ["LOGURU_LEVEL"] = level
+
+
+def _complete_yaml(incomplete: str):  # pragma: no cover
     for v in Path(".").glob(f"{incomplete}*.yaml"):
         yield str(v)
 
@@ -468,6 +491,8 @@ def pl_main(
     TRUE boolean values. Use `false` or `true` to explicitly set a boolean
     and `none`/`null`/`nul` to enforce `None`.
     """
+    _set_logger_level(verbose)
+
     PipelimeSymbolsHelper.set_extra_modules(extra_modules)
 
     if command_args is None:

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/commands/interfaces.py

@@ -819,7 +819,7 @@ class Interval(PydanticFieldWithDefaultMixin, pyd.BaseModel, extra="forbid"):
     """
 
     _default_type_description: t.ClassVar[t.Optional[str]] = "An interval of indexes."
-    _compact_form: t.ClassVar[t.Optional[str]] = "<start>[:<stop>]"
+    _compact_form: t.ClassVar[t.Optional[str]] = "[<start>][:<stop>]"
 
     start: t.Optional[int] = pyd.Field(
         None,
@@ -872,7 +872,7 @@ class ExtendedInterval(Interval):
     ``step`` indices or a string ``start:stop:step``.
     """
 
-    _compact_form: t.ClassVar[t.Optional[str]] = "<start>[:<stop>[:<step>]]"
+    _compact_form: t.ClassVar[t.Optional[str]] = "[<start>][:<stop>][:<step>]"
 
     step: t.Optional[int] = pyd.Field(
         None, description="The slice step, defaults to 1 (can be negative)."

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/commands/shell.py

@@ -50,8 +50,9 @@ class ShellCommand(PipelimeCommand, title="shell"):
 
     def _to_command_chunk(self, key: str, value: Any) -> str:
         cmd = ""
-        if isinstance(value, Sequence):
-            cmd += f" --{key} {value[0]} {self._to_command_chunk(key, value[1:])}"
+        if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
+            if value:
+                cmd += f" --{key} {value[0]} {self._to_command_chunk(key, value[1:])}"
         elif isinstance(value, Mapping):
             raise NotImplementedError("Mapping values are not supported")
         elif isinstance(value, bool):
@@ -68,8 +69,7 @@ class ShellCommand(PipelimeCommand, title="shell"):
         cmd = cmd.format(**args)
 
         for key in set(args.keys()).difference(fields):
-            value = args[key]
-            cmd += self._to_command_chunk(key, value)
+            cmd += self._to_command_chunk(key, args[key])
 
         with self.create_task(1, "Executing") as t:
             subprocess.run(cmd, shell=True)

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/commands/split_ops.py

@@ -202,7 +202,7 @@ class SplitCommand(PipelimeCommand, title="split"):
             range(len(split_sizes)), split_sizes, splits
         ):
             split_stop = split_start + split_length  # type: ignore
-            if split.output is not None:
+            if split.output is not None:  # pragma: no branch
                 seq = reader[split_start:split_stop]
                 seq = split.output.append_writer(seq)
                 self.grabber.grab_all(
@@ -376,7 +376,7 @@ class SplitByValueCommand(PipelimeCommand, title="split-value"):
 
             def __call__(self, x: Sample):
                 value = x.deep_get(self._value_key)
-                if value is not None:
+                if value is not None:  # pragma: no branch
                     value = self._value_to_str(value)
                     self._groups.setdefault(value, []).append(
                         int(x[self._idx_key]())  # type: ignore

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/commands/tempman.py

@@ -102,7 +102,7 @@ class TempCommand(PipelimeCommand, title="tmp"):
 
         to_delete = self._folders_to_delete()
 
-        if not self.force and to_delete:
+        if not self.force and to_delete:  # pragma: no cover
             table, total_size = self._paths_table(to_delete)
             table.title = f"\nTotal clean up: {self._human_size(total_size)}"
             rprint(table)
@@ -251,7 +251,7 @@ class TempCommand(PipelimeCommand, title="tmp"):
                 fp = os.path.join(dirpath, f)
                 try:
                     stat = os.stat(fp)
-                except OSError:
+                except OSError:  # pragma: no cover
                     continue
 
                 if stat.st_ino in seen:

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/piper/checkpoint.py

@@ -164,6 +164,7 @@ class LocalCheckpoint(Checkpoint, pyd.BaseModel):
 
             if self.try_link:
                 if source.is_dir():
+                    target.mkdir(parents=True, exist_ok=True)
                     for parent_dir, dirs, files in os.walk(source):
                         subdir = os.path.relpath(parent_dir, source)
                         trg_parent = os.path.join(target, subdir)
@@ -174,12 +175,12 @@ class LocalCheckpoint(Checkpoint, pyd.BaseModel):
                             trgf = os.path.join(trg_parent, f)
                             try:
                                 os.link(srcf, trgf)
-                            except Exception:
+                            except Exception:  # pragma: no cover
                                 shutil.copy2(srcf, trgf)
                 else:
                     try:
                         os.link(source, target)
-                    except Exception:
+                    except Exception:  # pragma: no cover
                         shutil.copy2(source, target)
             elif source.is_dir():
                 shutil.copytree(source, target)

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/piper/drawing/mermaid.py

@@ -16,7 +16,7 @@ from pipelime.piper.graph import (
 )
 
 
-class MermaidNodesGraphDrawer(NodesGraphDrawer):
+class MermaidNodesGraphDrawer(NodesGraphDrawer):  # pragma: no cover
     PRIVATE_CHARS = ["@"]
     SUBSTITUTION_CHAR = "_"
     OPERATION_NAME = "operation"

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/piper/executors/base.py

@@ -14,6 +14,7 @@ from pipelime.piper.progress.listener.factory import (
 )
 from pipelime.piper.progress.tracker.base import TrackedTask
 from pipelime.piper import PipelimeCommand
+from pipelime.utils.deprecated import deprecated
 
 
 class NodesGraphExecutor(ABC):
@@ -155,6 +156,7 @@ class NodesGraphExecutor(ABC):
         """
 
 
+@deprecated("Use a DirectTrackCallback instead, see RunCommandBase")
 class WatcherNodesGraphExecutor(NodesGraphExecutor):
     """Decorator for `NodesGraphExecutor` that wraps an existing executor.
 

{pipelime_python-1.9.0 → pipelime_python-1.9.1}/pipelime/piper/executors/factory.py

@@ -27,7 +27,7 @@ class NodesGraphExecutorFactory:
         )
         executor.task = task
 
-        if watch:
+        if watch:  # pragma: no cover
             executor = WatcherNodesGraphExecutor(
                 executor, watch if isinstance(watch, (str, Path)) else None
             )

pipelime_python-1.9.1/pipelime/utils/deprecated.py

@@ -0,0 +1,51 @@
+import functools
+import inspect
+import warnings
+
+
+def deprecated(reason):  # pragma: no cover
+    """Decorator to mark classes and functions as deprecated.
+    A warning will be emitted when the function is used or the class is created.
+    """
+
+    def _helper(wrapped, extra):
+        msg = (
+            "Call to deprecated "
+            f"{'class' if inspect.isclass(wrapped) else 'function'} "
+            f"{wrapped.__name__}{extra}"
+        )
+
+        def show_warning():
+            warnings.simplefilter("always", DeprecationWarning)
+            warnings.warn(
+                msg,
+                category=DeprecationWarning,
+                stacklevel=3,
+            )
+            warnings.simplefilter("default", DeprecationWarning)
+
+        if inspect.isclass(wrapped):
+            old_new1 = wrapped.__new__
+
+            def wrapped_new(cls, *args, **kwargs):
+                show_warning()
+                if old_new1 is object.__new__:
+                    return old_new1(cls)  # type: ignore
+                return old_new1(cls, *args, **kwargs)
+
+            wrapped.__new__ = staticmethod(functools.wraps(old_new1)(wrapped_new))  # type: ignore
+
+            return wrapped
+        else:
+
+            @functools.wraps(wrapped)
+            def wrapper(*args, **kwargs):
+                show_warning()
+                return wrapped(*args, **kwargs)
+
+            return wrapper
+
+    if isinstance(reason, (str, bytes)):
+        return functools.partial(_helper, extra=f" ({reason})")
+    else:
+        return _helper(reason, "")