zrb 0.0.45__py3-none-any.whl → 0.0.47__py3-none-any.whl

zrb/helper/env_map/fetch.py

@@ -0,0 +1,68 @@
+from typing import List, Mapping
+from ...task.base_task import Group, BaseTask
+from ...task_env.env import Env
+from ..string.jinja import is_probably_jinja
+
+
+def fetch_env_map_from_group(
+    env_map: Mapping[str, str], group: Group
+) -> Mapping[str, str]:
+    for task in group.tasks:
+        env_map = fetch_env_map_from_task(env_map, task)
+    for sub_group in group.children:
+        sub_env_map: Mapping[str, str] = fetch_env_map_from_group(
+            env_map, sub_group
+        )
+        env_map = cascade_env_map(env_map, sub_env_map)
+    return env_map
+
+
+def fetch_env_map_from_task(
+    env_map: Mapping[str, str], task: BaseTask
+):
+    task_env_map: Mapping[str, str] = {}
+    for env_file in task.env_files:
+        envs = env_file.get_envs()
+        task_env_map = add_envs_to_env_map(task_env_map, envs)
+    task_env_map = add_envs_to_env_map(task_env_map, task.envs)
+    env_map = cascade_env_map(env_map, task_env_map)
+    for upstream in task.upstreams:
+        task_env_map = fetch_env_map_from_task(env_map, upstream)
+    for checker in task.checkers:
+        task_env_map = fetch_env_map_from_task(env_map, checker)
+    return env_map
+
+
+def add_envs_to_env_map(
+    env_map: Mapping[str, str], envs: List[Env]
+) -> Mapping[str, str]:
+    for env in envs:
+        if env.os_name == '':
+            continue
+        env_name = get_env_name(env)
+        env_default = get_env_default(env)
+        env_map[env_name] = env_default
+    return env_map
+
+
+def cascade_env_map(
+    env_map: Mapping[str, str],
+    other_env_map: Mapping[str, str]
+) -> Mapping[str, str]:
+    for key, value in other_env_map.items():
+        if key in env_map:
+            continue
+        env_map[key] = value
+    return env_map
+
+
+def get_env_name(env: Env) -> str:
+    if env.os_name is None:
+        return env.name
+    return env.os_name
+
+
+def get_env_default(env: Env) -> str:
+    if is_probably_jinja(env.default):
+        return ''
+    return env.default

zrb/helper/file/copy_tree.py

@@ -1,6 +1,7 @@
 from typing import Iterable, Mapping, Optional
-from ..string.parse_replacement import parse_replacement
 from typeguard import typechecked
+from .text import read_text_file_async, write_text_file_async
+from ..string.parse_replacement import parse_replacement
 from ..log import logger
 
 import os
@@ -9,7 +10,7 @@ import fnmatch
 
 
 @typechecked
-def copy_tree(
+async def copy_tree(
     src: str,
     dst: str,
     replacements: Optional[Mapping[str, str]] = None,
@@ -29,15 +30,13 @@ def copy_tree(
             continue
         dst_name = os.path.join(dst, name)
         if os.path.isdir(src_name):
-            copy_tree(src_name, dst_name, replacements, excludes)
+            await copy_tree(src_name, dst_name, replacements, excludes)
             continue
         new_dst_name = parse_replacement(dst_name, replacements)
         shutil.copy2(src_name, new_dst_name)
         try:
-            with open(new_dst_name, 'r') as file:
-                file_content = file.read()
+            file_content = await read_text_file_async(new_dst_name)
             new_file_content = parse_replacement(file_content, replacements)
-            with open(new_dst_name, 'w') as file:
-                file.write(new_file_content)
+            await write_text_file_async(new_dst_name, new_file_content)
         except Exception:
             logger.error(f'Cannot parse file: {new_dst_name}')

zrb/helper/file/text.py

@@ -0,0 +1,20 @@
+import aiofiles
+
+
+async def read_text_file_async(file_name: str) -> str:
+    async with aiofiles.open(file_name, mode='r') as file:
+        content = await file.read()
+    return content
+
+
+async def write_text_file_async(file_name: str, content: str):
+    async with aiofiles.open(file_name, mode='w') as file:
+        await file.write(content)
+
+
+async def append_text_file_async(file_name: str, additional_content: str):
+    content = await read_text_file_async(file_name)
+    if content != '':
+        additional_content = '\n' + additional_content
+    new_content = content + additional_content
+    await write_text_file_async(file_name, new_content)

zrb/helper/string/jinja.py

@@ -0,0 +1,11 @@
+from typing import Any
+
+
+def is_probably_jinja(value: Any) -> bool:
+    if not isinstance(value, str):
+        return False
+    if '{{' in value and '}}' in value:
+        return True
+    if '{%' in value and '%}' in value:
+        return True
+    return False

zrb/task/base_task.py

@@ -2,23 +2,474 @@ from typing import (
     Any, Callable, Iterable, List, Mapping, Optional, TypeVar, Union
 )
 from typeguard import typechecked
-from .base_model import TaskModel
 from ..advertisement import advertisements
 from ..task_env.env import Env
 from ..task_env.env_file import EnvFile
-from ..task_group.group import Group
 from ..task_input.base_input import BaseInput
-from ..helper.accessories.color import colored
+from ..task_input._constant import RESERVED_INPUT_NAMES
+from ..helper.accessories.color import (
+    get_random_color, is_valid_color, colored
+)
+from ..helper.accessories.icon import get_random_icon
 from ..helper.advertisement import get_advertisement
 from ..helper.list.ensure_uniqueness import ensure_uniqueness
+from ..helper.log import logger
+from ..helper.render_data import DEFAULT_RENDER_DATA
 from ..helper.string.double_quote import double_quote
+from ..helper.string.conversion import (
+    to_cmd_name, to_variable_name, to_boolean
+)
+from ..helper.string.jinja import is_probably_jinja
 
 import asyncio
-import inspect
 import copy
+import datetime
+import inspect
+import jinja2
+import os
 import sys
+import time
+
+
+MAX_NAME_LENGTH = 20
+MULTILINE_INDENT = ' ' * 8
 
 TTask = TypeVar('TTask', bound='BaseTask')
+TGroup = TypeVar('TGroup', bound='Group')
+
+
+@typechecked
+class Group():
+    def __init__(
+        self,
+        name: str,
+        description: Optional[str] = None,
+        parent: Optional[TGroup] = None
+    ):
+        self.name = name
+        self.description = description
+        self.parent = parent
+        if parent is not None:
+            parent.children.append(self)
+        self.children: List[TGroup] = []
+        self.tasks: List[TTask] = []
+
+    def get_cmd_name(self) -> str:
+        return to_cmd_name(self.name)
+
+    def get_complete_name(self) -> str:
+        cmd_name = self.get_cmd_name()
+        if self.parent is None:
+            return cmd_name
+        parent_cmd_name = self.parent.get_complete_name()
+        return f'{parent_cmd_name} {cmd_name}'
+
+    def get_id(self) -> str:
+        group_id = self.get_cmd_name()
+        if self.parent is None:
+            return group_id
+        parent_group_id = self.parent.get_id()
+        return f'{parent_group_id} {group_id}'
+
+
+class AnyExtensionFileSystemLoader(jinja2.FileSystemLoader):
+    def get_source(self, environment, template):
+        for search_dir in self.searchpath:
+            file_path = os.path.join(search_dir, template)
+            if os.path.exists(file_path):
+                with open(file_path, 'r') as file:
+                    contents = file.read()
+                return contents, file_path, lambda: False
+        raise jinja2.TemplateNotFound(template)
+
+
+@typechecked
+class TimeTracker():
+
+    def __init__(self):
+        self._start_time: float = 0
+        self._end_time: float = 0
+
+    def start_timer(self):
+        self._start_time = time.time()
+
+    def end_timer(self):
+        self._end_time = time.time()
+
+    def get_elapsed_time(self) -> float:
+        return self._end_time - self._start_time
+
+
+@typechecked
+class AttemptTracker():
+
+    def __init__(self, retry: int = 2):
+        self.retry = retry
+        self._attempt: int = 1
+
+    def get_max_attempt(self) -> int:
+        return self.retry + 1
+
+    def get_attempt(self) -> int:
+        return self._attempt
+
+    def increase_attempt(self):
+        self._attempt += 1
+
+    def should_attempt(self) -> bool:
+        attempt = self.get_attempt()
+        max_attempt = self.get_max_attempt()
+        return attempt <= max_attempt
+
+    def is_last_attempt(self) -> bool:
+        attempt = self.get_attempt()
+        max_attempt = self.get_max_attempt()
+        return attempt >= max_attempt
+
+
+@typechecked
+class FinishTracker():
+
+    def __init__(self):
+        self._execution_queue: Optional[asyncio.Queue] = None
+        self._counter = 0
+
+    async def mark_start(self):
+        if self._execution_queue is None:
+            self._execution_queue = asyncio.Queue()
+        self._counter += 1
+
+    async def mark_as_done(self):
+        # Tracker might be started several times
+        # However, when the execution is marked as done, it applied globally
+        # Thus, we need to send event as much as the counter.
+        for i in range(self._counter):
+            await self._execution_queue.put(True)
+
+    async def is_done(self) -> bool:
+        while self._execution_queue is None:
+            await asyncio.sleep(0.05)
+        return await self._execution_queue.get()
+
+
+@typechecked
+class PidModel():
+
+    def __init__(self):
+        self.zrb_task_pid: int = os.getpid()
+
+    def set_task_pid(self, pid: int):
+        self.zrb_task_pid = pid
+
+    def get_task_pid(self) -> int:
+        return self.zrb_task_pid
+
+
+@typechecked
+class TaskModel(
+    PidModel, FinishTracker, AttemptTracker, TimeTracker
+):
+    def __init__(
+        self,
+        name: str,
+        group: Optional[Group] = None,
+        envs: Iterable[Env] = [],
+        env_files: Iterable[EnvFile] = [],
+        icon: Optional[str] = None,
+        color: Optional[str] = None,
+        retry: int = 2,
+    ):
+        # init properties
+        self.name = name
+        self.group = group
+        self.envs = envs
+        self.env_files = env_files
+        self.icon = icon
+        self.color = color
+        # init parent classes
+        PidModel.__init__(self)
+        FinishTracker.__init__(self)
+        TimeTracker.__init__(self)
+        non_negative_retry = self.ensure_non_negative(
+            retry, 'Find negative retry'
+        )
+        AttemptTracker.__init__(self, retry=non_negative_retry)
+        # init private properties
+        self._input_map: Mapping[str, Any] = {}
+        self._env_map: Mapping[str, str] = {}
+        self._complete_name: Optional[str] = None
+        self._filled_complete_name: Optional[str] = None
+        self._rendered_str: Mapping[str, str] = {}
+        self._is_keyval_set = False  # Flag
+        self._has_cli_interface = False
+        self._render_data: Optional[Mapping[str, Any]] = None
+        self._all_inputs: Optional[List[BaseInput]] = None
+
+    def get_icon(self) -> str:
+        if self.icon is None or self.icon == '':
+            self.icon = get_random_icon()
+        return self.icon
+
+    def get_color(self) -> str:
+        if self.color is None or not is_valid_color(self.color):
+            self.color = get_random_color()
+        return self.color
+
+    def get_cmd_name(self) -> str:
+        return to_cmd_name(self.name)
+
+    def ensure_non_negative(self, value: float, error_label: str) -> float:
+        if value < 0:
+            self.log_warn(f'{error_label}: {value}')
+            return 0
+        return value
+
+    def show_done_info(self):
+        complete_name = self._get_complete_name()
+        elapsed_time = self.get_elapsed_time()
+        message = '\n'.join([
+            f'{complete_name} completed in {elapsed_time} seconds',
+        ])
+        self.print_out_dark(message)
+        self.play_bell()
+
+    def log_debug(self, message: Any):
+        prefix = self._get_log_prefix()
+        colored_message = colored(
+            f'{prefix} • {message}', attrs=['dark']
+        )
+        logger.debug(colored_message)
+
+    def log_warn(self, message: Any):
+        prefix = self._get_log_prefix()
+        colored_message = colored(
+            f'{prefix} • {message}', attrs=['dark']
+        )
+        logger.warning(colored_message)
+
+    def log_info(self, message: Any):
+        prefix = self._get_log_prefix()
+        colored_message = colored(
+            f'{prefix} • {message}', attrs=['dark']
+        )
+        logger.info(colored_message)
+
+    def log_error(self, message: Any):
+        prefix = self._get_log_prefix()
+        colored_message = colored(
+            f'{prefix} • {message}', color='red', attrs=['bold']
+        )
+        logger.error(colored_message, exc_info=True)
+
+    def log_critical(self, message: Any):
+        prefix = self._get_log_prefix()
+        colored_message = colored(
+            f'{prefix} • {message}', color='red', attrs=['bold']
+        )
+        logger.critical(colored_message, exc_info=True)
+
+    def print_out(self, msg: Any):
+        prefix = self._get_colored_print_prefix()
+        print(f'🤖 ➜  {prefix} • {msg}'.rstrip(), file=sys.stderr)
+
+    def print_err(self, msg: Any):
+        prefix = self._get_colored_print_prefix()
+        print(f'🤖 ⚠  {prefix} • {msg}'.rstrip(), file=sys.stderr)
+
+    def print_out_dark(self, msg: Any):
+        self.print_out(colored(msg, attrs=['dark']))
+
+    def colored(self, text: str) -> str:
+        return colored(text, color=self.get_color())
+
+    def play_bell(self):
+        print('\a', end='', file=sys.stderr)
+
+    def _get_colored_print_prefix(self) -> str:
+        return self.colored(self._get_print_prefix())
+
+    def _get_print_prefix(self) -> str:
+        common_prefix = self._get_common_prefix(show_time=True)
+        icon = self.get_icon()
+        truncated_name = self._get_filled_complete_name()
+        return f'{common_prefix} • {icon} {truncated_name}'
+
+    def _get_log_prefix(self) -> str:
+        common_prefix = self._get_common_prefix(show_time=False)
+        icon = self.get_icon()
+        filled_name = self._get_filled_complete_name()
+        return f'{common_prefix} • {icon} {filled_name}'
+
+    def _get_common_prefix(self, show_time: bool) -> str:
+        attempt = self.get_attempt()
+        max_attempt = self.get_max_attempt()
+        pid = self.get_task_pid()
+        if show_time:
+            now = datetime.datetime.now().isoformat()
+            return f'{now} ⚙ {pid} ➤ {attempt} of {max_attempt}'
+        return f'⚙ {pid} ➤ {attempt} of {max_attempt}'
+
+    def _get_filled_complete_name(self) -> str:
+        if self._filled_complete_name is not None:
+            return self._filled_complete_name
+        complete_name = self._get_complete_name()
+        self._filled_complete_name = complete_name.rjust(MAX_NAME_LENGTH, ' ')
+        return self._filled_complete_name
+
+    def get_input_map(self) -> Mapping[str, Any]:
+        return self._input_map
+
+    def get_env_map(self) -> Mapping[str, str]:
+        return self._env_map
+
+    def _inject_env_map(
+        self, env_map: Mapping[str, str], override: bool = False
+    ):
+        for key, val in env_map.items():
+            if override or key not in self._env_map:
+                self._env_map[key] = val
+
+    def render_any(self, val: Any) -> Any:
+        if isinstance(val, str):
+            return self.render_str(val)
+        return val
+
+    def render_float(self, val: Union[str, float]) -> float:
+        if isinstance(val, str):
+            return float(self.render_str(val))
+        return val
+
+    def render_int(self, val: Union[str, int]) -> int:
+        if isinstance(val, str):
+            return int(self.render_str(val))
+        return val
+
+    def render_bool(self, val: Union[str, bool]) -> bool:
+        if isinstance(val, str):
+            return to_boolean(self.render_str(val))
+        return val
+
+    def render_str(self, val: str) -> str:
+        if val in self._rendered_str:
+            return self._rendered_str[val]
+        if not is_probably_jinja(val):
+            return val
+        template = jinja2.Template(val)
+        data = self._get_render_data()
+        self.log_debug(f'Render string template: {val}, with data: {data}')
+        try:
+            rendered_text = template.render(data)
+            self.log_debug(f'Get rendered result: {rendered_text}')
+        except Exception:
+            self.log_error(f'Fail to render "{val}" with data: {data}')
+        self._rendered_str[val] = rendered_text
+        return rendered_text
+
+    def render_file(self, location: str) -> str:
+        location_dir = os.path.dirname(location)
+        env = jinja2.Environment(
+            loader=AnyExtensionFileSystemLoader([location_dir])
+        )
+        template = env.get_template(location)
+        data = self._get_render_data()
+        data['TEMPLATE_DIR'] = location_dir
+        self.log_debug(f'Render template file: {template}, with data: {data}')
+        rendered_text = template.render(data)
+        self.log_debug(f'Get rendered result: {rendered_text}')
+        return rendered_text
+
+    def _get_render_data(self) -> Mapping[str, Any]:
+        if self._render_data is not None:
+            return self._render_data
+        render_data = dict(DEFAULT_RENDER_DATA)
+        render_data.update({
+            'env': self._env_map,
+            'input': self._input_map,
+        })
+        self._render_data = render_data
+        return render_data
+
+    def _get_multiline_repr(self, text: str) -> str:
+        lines_repr: Iterable[str] = []
+        lines = text.split('\n')
+        if len(lines) == 1:
+            return lines[0]
+        for index, line in enumerate(lines):
+            line_number_repr = str(index + 1).rjust(4, '0')
+            lines_repr.append(f'{MULTILINE_INDENT}{line_number_repr} | {line}')
+        return '\n' + '\n'.join(lines_repr)
+
+    async def _set_local_keyval(
+        self,
+        kwargs: Mapping[str, Any],
+        env_prefix: str = ''
+    ):
+        if self._is_keyval_set:
+            return True
+        self._is_keyval_set = True
+        # Add self.inputs to input_map
+        self.log_info('Set input map')
+        for task_input in self.get_all_inputs():
+            map_key = self._get_normalized_input_key(task_input.name)
+            self._input_map[map_key] = self.render_any(
+                kwargs.get(map_key, task_input.default)
+            )
+        self.log_debug(f'Input map: {self._input_map}')
+        # Construct envs based on self.env_files and self.envs
+        self.log_info('Merging task env_files and task envs')
+        envs: List[Env] = []
+        for env_file in self.env_files:
+            envs += env_file.get_envs()
+        envs += list(self.envs)
+        envs.reverse()
+        envs = ensure_uniqueness(envs, lambda x, y: x.name == y.name)
+        envs.reverse()
+        # Add envs to env_map
+        self.log_info('Set env map')
+        for task_env in envs:
+            env_name = task_env.name
+            if env_name in self._env_map:
+                continue
+            self._env_map[env_name] = self.render_any(
+                task_env.get(env_prefix)
+            )
+        self.log_info('Add os environment to env map')
+        for key in os.environ:
+            if key in self._env_map:
+                continue
+            self._env_map[key] = os.getenv(key, '')
+        self.log_debug(f'Env map: {self._env_map}')
+
+    def get_all_inputs(self) -> Iterable[BaseInput]:
+        # Override this method!!!
+        return self._all_inputs
+
+    def _get_normalized_input_key(self, key: str) -> str:
+        if key in RESERVED_INPUT_NAMES:
+            return key
+        return to_variable_name(key)
+
+    def _get_complete_name(self) -> str:
+        if self._complete_name is not None:
+            return self._complete_name
+        executable_prefix = ''
+        if self._has_cli_interface:
+            executable_prefix += self._get_executable_name() + ' '
+        cmd_name = self.get_cmd_name()
+        if self.group is None:
+            self._complete_name = f'{executable_prefix}{cmd_name}'
+            return self._complete_name
+        group_cmd_name = self.group.get_complete_name()
+        self._complete_name = f'{executable_prefix}{group_cmd_name} {cmd_name}'
+        return self._complete_name
+
+    def _get_executable_name(self) -> str:
+        if len(sys.argv) > 0 and sys.argv[0] != '':
+            return os.path.basename(sys.argv[0])
+        return 'zrb'
+
+    def set_has_cli_interface(self):
+        self._has_cli_interface = True
 
 
 @typechecked
@@ -64,6 +515,8 @@ class BaseTask(TaskModel):
         )
         if checking_interval == 0:
             checking_interval = 0.05 if len(checkers) == 0 else 0.1
+        if group is not None:
+            group.tasks.append(self)
         self.inputs = inputs
         self.description = description
         self.retry_interval = retry_interval

zrb/task/cmd_task.py

@@ -1,10 +1,9 @@
 from typing import Any, Callable, Iterable, Optional, Union
 from typeguard import typechecked
-from .base_task import BaseTask
+from .base_task import BaseTask, Group
 from ..task_env.env import Env
 from ..task_env.env_file import EnvFile
 from ..task_input.base_input import BaseInput
-from ..task_group.group import Group
 from ..config.config import default_shell
 
 import asyncio

zrb/task/decorator.py

@@ -5,7 +5,7 @@ from typeguard import typechecked
 from ..task_input.base_input import BaseInput
 from ..task_env.env import Env
 from ..task_env.env_file import EnvFile
-from ..task_group.group import Group
+from ..task.base_task import Group
 from ..action.runner import Runner
 from .base_task import BaseTask
 from .task import Task

zrb/task/docker_compose_task.py

@@ -1,14 +1,13 @@
 from typing import Any, Callable, Iterable, Mapping, Optional, Union
 from typeguard import typechecked
-from .base_task import BaseTask
+from .base_task import BaseTask, Group
 from .cmd_task import CmdTask
 from ..task_env.env import Env
 from ..task_env.env_file import EnvFile
 from ..task_input.base_input import BaseInput
-from ..task_group.group import Group
 from ..helper.string.double_quote import double_quote
-from ..helper.dockercompose.read import read_file as read_compose_file
-from ..helper.dockercompose.fetch_external_env import fetch_external_env
+from ..helper.docker_compose.file import read_compose_file
+from ..helper.docker_compose.fetch_external_env import fetch_external_env
 
 import os
 import pathlib

zrb/task/http_checker.py

@@ -1,10 +1,9 @@
 from typing import Any, Callable, Iterable, Optional, Union
 from typeguard import typechecked
 from http.client import HTTPConnection, HTTPSConnection
-from .base_task import BaseTask
+from .base_task import BaseTask, Group
 from ..task_env.env import Env
 from ..task_env.env_file import EnvFile
-from ..task_group.group import Group
 from ..task_input.base_input import BaseInput
 
 import asyncio