pipelime-python 1.8.1__tar.gz → 1.9.1__tar.gz

{pipelime_python-1.8.1 → pipelime_python-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pipelime-python
-Version: 1.8.1
+Version: 1.9.1
 Summary: Data workflows, cli and dataflow automation.
 Project-URL: Source, https://github.com/eyecan-ai/pipelime-python
 Project-URL: Issues, https://github.com/eyecan-ai/pipelime-python/issues
@@ -88,7 +88,7 @@ Maybe too much for you? No worries, `pipelime` is **modular** and you can just t
 - **easy dataflow**: `Piper` can manage and execute directed acyclic graphs (DAGs), giving back feedback on the progress through sockets or custom callbacks.
 - **configuration management**: `Choixe` is a simple and intuitive mini scripting language designed to ease the creation of configuration files with the help of variables, symbol importing, for loops, switch statements, parameter sweeps and more.
 - **command line interface**: `pipelime` can remove all the boilerplate code needed to create a beautiful CLI for you scripts and packages. You focus on *what matters* and we provide input parsing, advanced interfaces for complex arguments, automatic help generation, configuration management. Also, any `PipelimeCommand` can be used as a node in a dataflow for free!
-- **pydantic tools**: most of the classes in `pipelime` derive from [`pydantic.BaseModel`](https://docs.pydantic.dev/), so we have built some useful tools to, e.g., inspect their structure, auto-generate human-friendly documentation and more (including a wizard to help you writing input data to [deserialize](https://docs.pydantic.dev/usage/models/#helper-functions) any pydantic model).
+- **pydantic tools**: most of the classes in `pipelime` derive from [`pydantic.BaseModel`](https://docs.pydantic.dev/), so we have built some useful tools to, e.g., inspect their structure, auto-generate human-friendly documentation and more (including a TUI to help you writing input data to [deserialize](https://docs.pydantic.dev/usage/models/#helper-functions) any pydantic model).
 
 ---
 

{pipelime_python-1.8.1 → pipelime_python-1.9.1}/README.md

@@ -20,7 +20,7 @@ Maybe too much for you? No worries, `pipelime` is **modular** and you can just t
 - **easy dataflow**: `Piper` can manage and execute directed acyclic graphs (DAGs), giving back feedback on the progress through sockets or custom callbacks.
 - **configuration management**: `Choixe` is a simple and intuitive mini scripting language designed to ease the creation of configuration files with the help of variables, symbol importing, for loops, switch statements, parameter sweeps and more.
 - **command line interface**: `pipelime` can remove all the boilerplate code needed to create a beautiful CLI for you scripts and packages. You focus on *what matters* and we provide input parsing, advanced interfaces for complex arguments, automatic help generation, configuration management. Also, any `PipelimeCommand` can be used as a node in a dataflow for free!
-- **pydantic tools**: most of the classes in `pipelime` derive from [`pydantic.BaseModel`](https://docs.pydantic.dev/), so we have built some useful tools to, e.g., inspect their structure, auto-generate human-friendly documentation and more (including a wizard to help you writing input data to [deserialize](https://docs.pydantic.dev/usage/models/#helper-functions) any pydantic model).
+- **pydantic tools**: most of the classes in `pipelime` derive from [`pydantic.BaseModel`](https://docs.pydantic.dev/), so we have built some useful tools to, e.g., inspect their structure, auto-generate human-friendly documentation and more (including a TUI to help you writing input data to [deserialize](https://docs.pydantic.dev/usage/models/#helper-functions) any pydantic model).
 
 ---
 

{pipelime_python-1.8.1 → pipelime_python-1.9.1}/pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "1.8.1"
+__version__ = "1.9.1"

{pipelime_python-1.8.1 → pipelime_python-1.9.1}/pipelime/choixe/utils/imports.py

@@ -103,13 +103,15 @@ def import_module_from_file(
     Args:
         module_file_path (Union[str, Path]): the path to the `.py` module file.
         cwd (Optional[Path], optional): the folder to use for relative module import.
-            If None, the file parent folder will be used. Defaults to None.
+            Defaults to None.
+        register_pl_module (bool, optional): whether to register the module with
+            PipelimeSymbolsHelper. Defaults to True.
 
     Raises:
-        ImportError: _description_
+        ImportError: if the module cannot be imported.
 
     Returns:
-        ModuleType: _description_
+        ModuleType: the imported module.
     """
     from pipelime.cli.utils import PipelimeSymbolsHelper
 
@@ -130,6 +132,16 @@ def import_module_from_file(
 def import_module_from_class_path(
     module_class_path: str, register_pl_module: bool = True
 ) -> ModuleType:
+    """Import a python module from a python class path.
+
+    Args:
+        module_class_path (str): the dot-separated path to the module class.
+        register_pl_module (bool, optional): whether to register the module with
+            PipelimeSymbolsHelper. Defaults to True.
+
+    Returns:
+        ModuleType: the imported module.
+    """
     from pipelime.cli.utils import PipelimeSymbolsHelper
 
     m = importlib.import_module(module_class_path)
@@ -141,6 +153,19 @@ def import_module_from_class_path(
 def import_module_from_code(
     module_code: str, register_pl_module: bool = True
 ) -> ModuleType:
+    """Dynamically imports a Python module from its source code.
+
+    Args:
+        module_code (str): the python source code of the module to import.
+        register_pl_module (bool, optional): whether to register the module with
+            PipelimeSymbolsHelper. Defaults to True.
+
+    Raises:
+        ImportError: if the module cannot be imported.
+
+    Returns:
+        ModuleType: the imported module.
+    """
     import hashlib
 
     from pipelime.cli.utils import PipelimeSymbolsHelper
@@ -181,7 +206,23 @@ def import_module(
     cwd: Optional[Path] = None,
     register_pl_module: bool = True,
 ) -> ModuleType:
-    err_msgs = []
+    """Import a module from a file, a class path, or its source code.
+
+    Args:
+        module_file_or_class_path_or_code (str): the path to the module file, the
+            python class path, or its python code.
+        cwd (Optional[Path], optional): the current working directory for relative
+            imports. Defaults to None.
+        register_pl_module (bool, optional): whether to register the module with
+            PipelimeSymbolsHelper. Defaults to True.
+
+    Raises:
+        ImportError: If the module cannot be imported.
+
+    Returns:
+        ModuleType: The imported module.
+    """
+    err_msgs = [""]
 
     if module_file_or_class_path_or_code.endswith(".py"):
         try:
@@ -189,28 +230,25 @@ def import_module(
                 module_file_or_class_path_or_code, cwd, register_pl_module
             )
         except Exception as e:
-            err_msgs.append(str(e))
+            err_msgs.append(f"from file: {e}")
     try:
         return import_module_from_class_path(
             module_file_or_class_path_or_code, register_pl_module
         )
     except Exception as e:
-        err_msgs.append(str(e))
+        err_msgs.append(f"from classpath: {e}")
 
     try:
         return import_module_from_code(
             module_file_or_class_path_or_code, register_pl_module
         )
     except Exception as e:
-        err_msgs.append(str(e))
+        err_msgs.append(f"from code: {e}")
 
     raise ImportError(
         "Cannot import:\n"
         f"  `{module_file_or_class_path_or_code}`\n"
-        "Errors:\n"
-        f"  from file: {err_msgs[0]}\n"
-        f"  from classpath: {err_msgs[1]}\n"
-        f"  from code: {err_msgs[2]}"
+        "Possible causes:" + "\n  ".join(err_msgs)
     )
 
 

{pipelime_python-1.8.1 → pipelime_python-1.9.1}/pipelime/choixe/utils/io.py

@@ -111,9 +111,9 @@ class PipelimeTmp:
 
         user_dirs: Dict[str, List[str]] = {}
         for d in iglob(str(PipelimeTmp.base_dir() / (PipelimeTmp.prefix() + "*"))):
-            if os.path.isdir(d):
+            if os.path.isdir(d):  # pragma: no branch
                 names = os.path.basename(d).split("-")
-                if len(names) >= 3:
+                if len(names) >= 3:  # pragma: no branch
                     user_dirs.setdefault(names[2], []).append(d)
         return user_dirs
 
@@ -135,7 +135,8 @@ class PipelimeTemporaryDirectory:
         import shutil
 
         def onerror(func, path, exc_info):
-            if issubclass(exc_info[0], PermissionError):
+            # this code comes from TemporaryDirectory
+            if issubclass(exc_info[0], PermissionError):  # pragma: no cover
 
                 def resetperms(path):
                     try:

{pipelime_python-1.8.1 → pipelime_python-1.9.1}/pipelime/choixe/visitors/processor.py

@@ -1,6 +1,6 @@
 import os
 import uuid
-from copy import deepcopy
+from copy import copy
 from dataclasses import dataclass
 from datetime import datetime
 from itertools import product
@@ -12,7 +12,7 @@ import pydash as py_
 import pipelime.choixe.ast.nodes as ast
 from pipelime.choixe.ast.parser import parse
 from pipelime.choixe.utils.imports import import_symbol
-from pipelime.choixe.utils.io import load, PipelimeTmp
+from pipelime.choixe.utils.io import PipelimeTmp, load
 from pipelime.choixe.utils.rand import rand
 from pipelime.choixe.visitors.unparser import unparse
 
@@ -62,11 +62,37 @@ class Processor(ast.NodeVisitor):
         return list(product(*branches))
 
     def _repeat(self, data: Any, n: int) -> List[Any]:
+        # Avoid repeating if n is 1, skipping all the copy/deepcopy logic
+        if n == 1:
+            return data
+
         new_data = []
         for _ in range(n):
-            new_data.extend(deepcopy(data))
+            #! This used to work, but it fails when some data in the tree is not
+            #! serializable. The proposed workaround might break stuff in the future,
+            #! So I will leave this here for now.
+            # new_data.extend(deepcopy(data))
+
+            #! Simply copying the list e.g. `new_data.extend(copy(data))`
+            #! does not work because we need a true copy of the inner containers as
+            #! well, otherwise, any modification to the inner containers will be
+            #! reflected in all the branches.
+
+            #! The following code is a workaround that works for now, but it might
+            #! break in the future. It duplicates the data only for the first level,
+            #! switching to shallow copy for the inner containers.
+            new_data.extend([copy(x) for x in data])
         return new_data
 
+    def _handle_missing_var(self, varname: str) -> Any:
+        raise ChoixeProcessingError(f"Variable `{varname}` not found")
+
+    def _handle_missing_for(self, varname: str) -> Any:
+        raise ChoixeProcessingError(f"Loop variable `{varname}` not found")
+
+    def _handle_missing_switch(self, varname: str) -> Any:
+        raise ChoixeProcessingError(f"Switch variable `{varname}` not found")
+
     def visit_dict(self, node: ast.DictNode) -> List[Dict]:
         data = [{}]
         for k, v in node.nodes.items():
@@ -135,7 +161,7 @@ class Processor(ast.NodeVisitor):
                 found = True
 
             if not found:
-                raise ChoixeProcessingError(f"Variable not found: `{id_}`")
+                var_value = self._handle_missing_var(id_)
 
             # Recursively process the variable value
             re_parsed = parse(var_value)
@@ -192,7 +218,10 @@ class Processor(ast.NodeVisitor):
 
     def visit_for(self, node: ast.ForNode) -> List[Any]:
         if isinstance(node.iterable.data, str):
-            iterable = py_.get(self._context, node.iterable.data)
+            if not py_.has(self._context, node.iterable.data):
+                iterable = self._handle_missing_for(node.iterable.data)
+            else:
+                iterable = py_.get(self._context, node.iterable.data)
         else:
             iterable = node.iterable.data
 
@@ -200,15 +229,10 @@ class Processor(ast.NodeVisitor):
             iterable = list(range(iterable))
 
         if not isinstance(iterable, Iterable):
-            if isinstance(node.iterable.data, str) and not py_.has(
-                self._context, node.iterable.data
-            ):
-                raise ChoixeProcessingError(
-                    f"Loop variable `{node.iterable.data}` not found in context"
-                )
             raise ChoixeProcessingError(
                 f"Loop variable `{node.iterable.data}` is not iterable"
             )
+
         id_ = uuid.uuid1().hex if node.identifier is None else str(node.identifier.data)
         prev_loop = self._current_loop
         self._current_loop = id_
@@ -254,13 +278,10 @@ class Processor(ast.NodeVisitor):
         for branch in all_branches:
             varname = branch[0]
 
-            # If the variable is not in the context, raise an error
             if not py_.has(self._context, varname):
-                msg = f"Switch variable `{varname}` not found in context"
-                raise ChoixeProcessingError(msg)
-
-            # Get the value of the variable
-            value = py_.get(self._context, varname)
+                value = self._handle_missing_switch(varname)
+            else:
+                value = py_.get(self._context, varname)
 
             # Match the value to the correct case
             for i in range(len(node.cases)):
@@ -275,7 +296,8 @@ class Processor(ast.NodeVisitor):
                     branches.append(branch[i + 1 + len(node.cases)])
                     break
 
-            # If no case matched, use the default if available, otherwise raise an error
+            # If no case matched, use the default if available
+            # otherwise raise an error
             else:
                 if node.default is not None:
                     branches.append(branch[-1])
@@ -372,5 +394,9 @@ def process(
         Any: The list of all possible outcomes. If branching is disabled, the list will
             have length 1.
     """
-    processor = Processor(context=context, cwd=cwd, allow_branching=allow_branching)
+    processor = Processor(
+        context=context,
+        cwd=cwd,
+        allow_branching=allow_branching,
+    )
     return node.accept(processor)

pipelime_python-1.9.1/pipelime/choixe/visitors/processor_ui.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from pipelime.choixe.visitors.processor import ChoixeProcessingError, Processor
+from pipelime.cli.utils import get_user_input
+
+
+class ProcessorUi(Processor):
+    """Processor that asks the user to fill the variables missing in the context."""
+
+    def __init__(
+        self,
+        context: Optional[Dict[str, Any]] = None,
+        cwd: Optional[Path] = None,
+        allow_branching: bool = True,
+        ask_missing_vars: bool = False,
+    ) -> None:
+        """Constructor for `ProcessorUi`
+
+        Args:
+            context (Optional[Dict[str, Any]], optional): A data structure containing
+                the values that will replace the variable nodes. Defaults to None.
+            cwd (Optional[Path], optional): current working directory used for relative
+                imports. If set to None, the `os.getcwd()` will be used.
+                Defaults to None.
+            allow_branching (bool, optional): Set to False to disable processing on
+                branching nodes, like sweeps. All branching nodes will be simply
+                unparsed. Defaults to True.
+            ask_missing_vars (bool, optional): Set to True to allow the user to fill
+                the variables missing in the context.
+        """
+        super().__init__(context, cwd, allow_branching)
+        self._ask_missing_vars = ask_missing_vars
+        self._user_defined_vars: Dict[str, Any] = {}
+
+    def _handle_missing_var(self, varname: str) -> Any:
+        if self._ask_missing_vars:
+            msg = f"Enter value for [yellow]{varname}[/yellow]"
+            value = get_user_input(msg)
+            self._user_defined_vars[varname] = value
+            self._context[varname] = value
+            return value
+        else:
+            raise ChoixeProcessingError(f"Variable not found: `{varname}`")
+
+    def _handle_missing_for(self, varname: str) -> Any:
+        if self._ask_missing_vars:
+            msg = f"Enter value for [yellow]{varname}[/yellow]"
+            iterable = get_user_input(msg)
+            self._user_defined_vars[varname] = iterable
+            self._context[varname] = iterable
+            return iterable
+        else:
+            raise ChoixeProcessingError(f"Loop variable `{varname}` not found")
+
+    def _handle_missing_switch(self, varname: str) -> Any:
+        if self._ask_missing_vars:
+            msg = f"Enter value for [yellow]{varname}[/yellow]"
+            value = get_user_input(msg)
+            self._user_defined_vars[varname] = value
+            self._context[varname] = value
+            return value
+        else:
+            raise ChoixeProcessingError(f"Switch variable `{varname}` not found")

{pipelime_python-1.8.1 → pipelime_python-1.9.1}/pipelime/choixe/xconfig.py

@@ -9,8 +9,8 @@ from schema import Schema
 
 from pipelime.choixe.ast.nodes import Node
 from pipelime.choixe.ast.parser import parse
-from pipelime.choixe.utils.io import dump, load
 from pipelime.choixe.utils.common import deep_set_
+from pipelime.choixe.utils.io import dump, load
 from pipelime.choixe.visitors import Inspection, decode, inspect, process, walk
 
 
@@ -224,15 +224,22 @@ class XConfig(Box):
         return walk(self.parse())
 
     def _process(
-        self, context: Optional[Dict[str, Any]] = None, allow_branching: bool = True
+        self,
+        context: Optional[Dict[str, Any]] = None,
+        allow_branching: bool = True,
     ) -> List[XConfig]:
-        data = self._unsafe_process(context=context, allow_branching=allow_branching)
+        data = self._unsafe_process(
+            context=context,
+            allow_branching=allow_branching,
+        )
         return [
             XConfig(data=x, cwd=self.get_cwd(), schema=self.get_schema()) for x in data
         ]
 
     def _unsafe_process(
-        self, context: Optional[Dict[str, Any]] = None, allow_branching: bool = True
+        self,
+        context: Optional[Dict[str, Any]] = None,
+        allow_branching: bool = True,
     ) -> List[Any]:
         return process(
             self.parse(),
@@ -251,7 +258,10 @@ class XConfig(Box):
         Returns:
             XConfig: The processed XConfig.
         """
-        return self._process(context=context, allow_branching=False)[0]
+        return self._process(
+            context=context,
+            allow_branching=False,
+        )[0]
 
     def unsafe_process(self, context: Optional[Dict[str, Any]] = None) -> Any:
         """Process this XConfig without branching and return the result, without
@@ -267,7 +277,10 @@ class XConfig(Box):
         Returns:
             Any: The processed result.
         """
-        return self._unsafe_process(context=context, allow_branching=False)[0]
+        return self._unsafe_process(
+            context=context,
+            allow_branching=False,
+        )[0]
 
     def process_all(self, context: Optional[Dict[str, Any]] = None) -> List[XConfig]:
         """Process this XConfig with branching.
@@ -279,7 +292,10 @@ class XConfig(Box):
         Returns:
             List[XConfig]: A list of all processing outcomes.
         """
-        return self._process(context=context, allow_branching=True)
+        return self._process(
+            context=context,
+            allow_branching=True,
+        )
 
     def unsafe_process_all(self, context: Optional[Dict[str, Any]] = None) -> List[Any]:
         """Process this XConfig with branching and return the results, without
@@ -295,7 +311,10 @@ class XConfig(Box):
         Returns:
             List[Any]: A list of all processing outcomes.
         """
-        return self._unsafe_process(context=context, allow_branching=True)
+        return self._unsafe_process(
+            context=context,
+            allow_branching=True,
+        )
 
     def inspect(self) -> Inspection:
         return inspect(self.parse(), cwd=self.get_cwd())