annet 0.16.35__py3-none-any.whl → 1.0.0__py3-none-any.whl

annet/adapters/fetchers/stub/fetcher.py

@@ -9,11 +9,17 @@ class StubFetcher(Fetcher, AdapterWithConfig):
     def with_config(cls, **kwargs: Dict[str, Any]) -> Fetcher:
         return cls(**kwargs)
 
-    def fetch_packages(self, devices: List[Device],
-                       processes: int = 1, max_slots: int = 0):
+    async def fetch_packages(self,
+                             devices: list[Device],
+                             processes: int = 1,
+                             max_slots: int = 0,
+                             ) -> tuple[dict[Device, str], dict[Device, Any]]:
         raise NotImplementedError()
 
-    def fetch(self, devices: List[Device],
-              files_to_download: Dict[str, List[str]] = None,
-              processes: int = 1, max_slots: int = 0):
+    async def fetch(self,
+                    devices: list[Device],
+                    files_to_download: dict[str, list[str]] | None = None,
+                    processes: int = 1,
+                    max_slots: int = 0,
+                    ):
         raise NotImplementedError()

annet/api/__init__.py

@@ -22,6 +22,8 @@ from typing import (
 )
 
 import colorama
+import annet.deploy
+import annet.deploy_ui
 import annet.lib
 from annet.annlib import jsontools
 from annet.annlib.netdev.views.hardware import HardwareView
@@ -262,7 +264,7 @@ def patch(args: cli_args.ShowPatchOptions, loader: ann_gen.Loader):
     global live_configs  # pylint: disable=global-statement
     if args.config == "running":
         fetcher = annet.deploy.get_fetcher()
-        live_configs = fetcher.fetch(loader.devices, processes=args.parallel)
+        live_configs = annet.lib.do_async(fetcher.fetch(loader.devices, processes=args.parallel))
     stdin = args.stdin(filter_acl=args.filter_acl, config=args.config)
 
     filterer = filtering.filterer_connector.get()
@@ -567,7 +569,7 @@ class Deployer:
             if not diff_obj:
                 self.empty_diff_hostnames.update(dev.hostname for dev in devices)
             if not self.args.no_ask_deploy:
-                # разобъем список устройств на несколько линий
+                # разобьём список устройств на несколько линий
                 dest_name = ""
                 try:
                     _, term_columns_str = os.popen("stty size", "r").read().split()
@@ -596,18 +598,18 @@ class Deployer:
         return diff_lines
 
     def ask_deploy(self) -> str:
-        return self._ask("y", annet.deploy.AskConfirm(
+        return self._ask("y", annet.deploy_ui.AskConfirm(
             text="\n".join(self.diff_lines()),
             alternative_text="\n".join(self.cmd_lines),
         ))
 
     def ask_rollback(self) -> str:
-        return self._ask("n", annet.deploy.AskConfirm(
+        return self._ask("n", annet.deploy_ui.AskConfirm(
             text="Execute rollback?\n",
             alternative_text="",
         ))
 
-    def _ask(self, default_ans: str, ask: annet.deploy.AskConfirm) -> str:
+    def _ask(self, default_ans: str, ask: annet.deploy_ui.AskConfirm) -> str:
         # если filter_acl из stdin то с ним уже не получится работать как с терминалом
         ans = default_ans
         if not self.args.no_ask_deploy:
@@ -666,11 +668,22 @@ def deploy(
     filterer: Filterer,
     fetcher: Fetcher,
     deploy_driver: DeployDriver,
+) -> ExitCode:
+    return annet.lib.do_async(adeploy(args, loader, deployer, filterer, fetcher, deploy_driver))
+
+
+async def adeploy(
+    args: cli_args.DeployOptions,
+    loader: ann_gen.Loader,
+    deployer: Deployer,
+    filterer: Filterer,
+    fetcher: Fetcher,
+    deploy_driver: DeployDriver,
 ) -> ExitCode:
     """ Сгенерировать конфиг для устройств и задеплоить его """
     ret: ExitCode = 0
     global live_configs  # pylint: disable=global-statement
-    live_configs = fetcher.fetch(devices=loader.devices, processes=args.parallel)
+    live_configs = await fetcher.fetch(devices=loader.devices, processes=args.parallel)
     pool = ann_gen.OldNewParallel(args, loader, filterer)
 
     for res in pool.generated_configs(loader.devices):
@@ -687,7 +700,18 @@ def deploy(
         ans = deployer.ask_deploy()
         if ans != "y":
             return 2 ** 2
-        result = annet.lib.do_async(deploy_driver.bulk_deploy(deploy_cmds, args))
+        progress_bar = None
+        if sys.stdout.isatty() and not args.no_progress:
+            progress_bar = annet.deploy_ui.ProgressBars(odict([(device.fqdn, {}) for device in deploy_cmds]))
+            progress_bar.init()
+            with progress_bar:
+                progress_bar.start_terminal_refresher()
+                result = await deploy_driver.bulk_deploy(deploy_cmds, args, progress_bar=progress_bar)
+                await progress_bar.wait_for_exit()
+            progress_bar.screen.clear()
+            progress_bar.stop_terminal_refresher()
+        else:
+            result = await deploy_driver.bulk_deploy(deploy_cmds, args)
 
     rolled_back = False
     rollback_cmds = {deployer.fqdn_to_device[x]: cc for x, cc in result.original_states.items() if cc}
@@ -695,7 +719,7 @@ def deploy(
         ans = deployer.ask_rollback()
         if rollback_cmds and ans == "y":
             rolled_back = True
-            annet.lib.do_async(deploy_driver.bulk_deploy(rollback_cmds, args))
+            await deploy_driver.bulk_deploy(rollback_cmds, args)
 
     if not args.no_check_diff and not rolled_back:
         deployer.check_diff(result, loader)

annet/deploy.py

@@ -1,37 +1,50 @@
 # pylint: disable=unused-argument
-
-
 import abc
 import itertools
-import re
 from collections import namedtuple
-from contextlib import contextmanager
 from typing import Dict, List, Optional, Any, OrderedDict, Tuple, Type
 
 from contextlog import get_logger
 
-from annet import text_term_format
 from annet.annlib.command import Command, Question, CommandList
 from annet.annlib.netdev.views.hardware import HardwareView
 from annet.annlib.rbparser.deploying import MakeMessageMatcher, Answer
 from annet.cli_args import DeployOptions
 from annet.connectors import Connector, get_connector_from_config
-from annet.output import TextArgs
 from annet.rulebook import get_rulebook, deploying
 from annet.storage import Device
 
 
-NCURSES_SIZE_T = 2 ** 15 - 1
+_DeployResultBase = namedtuple("_DeployResultBase", ("hostnames", "results", "durations", "original_states"))
 
 
-_DeployResultBase = namedtuple("_DeployResultBase", ("hostnames", "results", "durations", "original_states"))
+class ProgressBar(abc.ABC):
+    @abc.abstractmethod
+    def set_content(self, tile_name: str, content: str):
+        ...
+
+    @abc.abstractmethod
+    def set_progress(self,
+                     tile_name: str,
+                     iteration: int,
+                     total: int,
+                     prefix: str = "",
+                     suffix: str = "",
+                     fill: str = "",
+                     error: bool = False,
+                     ):
+        ...
+
+    @abc.abstractmethod
+    def set_exception(self, tile_name: str, cmd_exc: str, last_cmd: str, progress_max: int):
+        ...
 
 
 class DeployResult(_DeployResultBase):  # noqa: E302
-    def add_results(self, results: Dict[str, Optional[Exception]]) -> None:
-        for hostname, result in results.items():
+    def add_results(self, results: dict[str, tuple[list[str], list[Exception]]]) -> None:
+        for hostname, (excs, result) in results.items():
             self.hostnames.append(hostname)
-            self.results[hostname] = result
+            self.results[hostname] = excs
             self.durations[hostname] = 0.0
             self.original_states[hostname] = None
 
@@ -64,14 +77,20 @@ driver_connector = _DriverConnector()
 
 class Fetcher(abc.ABC):
     @abc.abstractmethod
-    def fetch_packages(self, devices: List[Device],
-                       processes: int = 1, max_slots: int = 0) -> Tuple[Dict[Device, str], Dict[Device, Any]]:
+    async def fetch_packages(self,
+                             devices: list[Device],
+                             processes: int = 1,
+                             max_slots: int = 0,
+                             ) -> tuple[dict[Device, str], dict[Device, Any]]:
         pass
 
     @abc.abstractmethod
-    def fetch(self, devices: List[Device],
-              files_to_download: Dict[str, List[str]] = None,
-              processes: int = 1, max_slots: int = 0):
+    async def fetch(self,
+                    devices: list[Device],
+                    files_to_download: dict[str, list[str]] | None = None,
+                    processes: int = 1,
+                    max_slots: int = 0,
+                    ):
         pass
 
 
@@ -83,7 +102,7 @@ def get_fetcher() -> Fetcher:
 
 class DeployDriver(abc.ABC):
     @abc.abstractmethod
-    async def bulk_deploy(self, deploy_cmds: dict, args: DeployOptions) -> DeployResult:
+    async def bulk_deploy(self, deploy_cmds: dict, args: DeployOptions, progress_bar: ProgressBar | None = None) -> DeployResult:
         pass
 
     @abc.abstractmethod
@@ -106,7 +125,7 @@ def get_deployer() -> DeployDriver:
 
 
 # ===
-def scrub_config(text, breed):
+def scrub_config(text: str, breed: str) -> str:
     return text
 
 
@@ -114,331 +133,6 @@ def show_bulk_report(hostnames, results, durations, log_dir):
     pass
 
 
-class AskConfirm:
-    CUT_WARN_MSG = "WARNING: the text was cut because of curses limits."
-
-    def __init__(self, text: str, text_type="diff", alternative_text: str = "",
-                 alternative_text_type: str = "diff", allow_force_yes: bool = False):
-        self.text = [text, text_type]
-        self.alternative_text = [alternative_text, alternative_text_type]
-        self.color_to_curses: Dict[Optional[str], int] = {}
-        self.lines: Dict[int, List[TextArgs]] = {}
-        self.rows = None
-        self.cols = None
-        self.top = 0
-        self.left = 0
-        self.pad = None
-        self.screen = None
-        self.found_pos = {}
-        self.curses_lines = None
-        self.debug_prompt = TextArgs("")
-        self.page_position = TextArgs("")
-        s_force = "/f" if allow_force_yes else ""
-        self.prompt = [
-            TextArgs("Execute these commands? [Y%s/q] (/ - search, a - patch/cmds)" % s_force, "blue", offset=0),
-            self.page_position,
-            self.debug_prompt]
-
-    def _parse_text(self):
-        txt = self.text[0]
-        txt_split = txt.splitlines()
-        # curses pad, который тут используется, имеет ограничение на количество линий
-        if (len(txt_split) + 1) >= NCURSES_SIZE_T:  # +1 для того чтобы курсор можно было переместить на пустую строку
-            del txt_split[NCURSES_SIZE_T - 3:]
-            txt_split.insert(0, self.CUT_WARN_MSG)
-            txt_split.append(self.CUT_WARN_MSG)
-            txt = "\n".join(txt_split)
-        self.rows = len(txt_split)
-        self.cols = max(len(line) for line in txt_split)
-        res = text_term_format.curses_format(txt, self.text[1])
-        self.lines = res
-
-    def _update_search_pos(self, expr):
-        self.found_pos = {}
-        if not expr:
-            return
-        try:
-            expr = re.compile(expr)
-        except Exception:
-            return None
-        lines = self.text[0].splitlines()
-        for (line_no, line) in enumerate(lines):
-            for match in re.finditer(expr, line):
-                if line_no not in self.found_pos:
-                    self.found_pos[line_no] = []
-                self.found_pos[line_no].append(TextArgs(match.group(0), "highlight", match.start()))
-
-    def _init_colors(self):
-        self.color_to_curses = init_colors()
-
-    def _init_pad(self):
-        import curses
-
-        with self._store_xy():
-            self.pad = curses.newpad(self.rows + 1, self.cols)
-            self.pad.keypad(True)  # accept arrow keys
-            self._render_to_pad(self.lines)
-
-    def _render_to_pad(self, lines: dict):
-        """
-        Рендерим данный на pad
-        :param lines: словарь проиндексированный по номерам линий
-        :return:
-        """
-        with self._store_xy():
-            for line_no, line_data in sorted(lines.items()):
-                line_pos_calc = 0
-                for line_part in line_data:
-                    if line_part.offset is not None:
-                        line_pos = line_part.offset
-                    else:
-                        line_pos = line_pos_calc
-                    if line_part.color:
-                        self.pad.addstr(line_no, line_pos, line_part.text, self.color_to_curses[line_part.color])
-                    else:
-                        self.pad.addstr(line_no, line_pos, line_part.text)
-                    line_pos_calc += len(line_part.text)
-
-    def _add_prompt(self):
-        for prompt_part in self.prompt:
-            if not prompt_part:
-                continue
-            if prompt_part.offset is None:
-                offset = 0
-            else:
-                offset = prompt_part.offset
-            self.screen.addstr(self.curses_lines - 1, offset, prompt_part.text, self.color_to_curses[prompt_part.color])
-
-    def _clear_prompt(self):
-        with self._store_xy():
-            self.screen.move(self.curses_lines - 1, 0)
-            self.screen.clrtoeol()
-
-    def show(self):
-        self._add_prompt()
-        self.screen.refresh()
-        size = self.screen.getmaxyx()
-        self.pad.refresh(self.top, self.left, 0, 0, size[0] - 2, size[1] - 2)
-
-    @contextmanager
-    def _store_xy(self):
-        if self.pad is not None:
-            current_y, current_x = self.pad.getyx()
-            yield current_y, current_x
-            max_y, max_x = self.pad.getmaxyx()
-            current_y = min(max_y - 1, current_y)
-            current_x = min(max_x - 1, current_x)
-
-            self.pad.move(current_y, current_x)
-        else:
-            yield
-
-    def search_next(self, prev=False):
-        to = None
-        current_y, current_x = self.pad.getyx()
-        if prev:
-            for line_index in sorted(self.found_pos, reverse=True):
-                for text_args in self.found_pos[line_index]:
-                    if line_index > current_y:
-                        continue
-
-                    if line_index < current_y or line_index == current_y and text_args.offset < current_x:
-                        to = line_index, text_args.offset
-                        break
-                if to:
-                    break
-        else:
-            for line_index in sorted([i for i in self.found_pos if i >= current_y]):
-                for text_args in self.found_pos[line_index]:
-                    if line_index > current_y or line_index == current_y and text_args.offset > current_x:
-                        to = line_index, text_args.offset
-                        break
-                if to:
-                    break
-        if to:
-            return to[0] - current_y, to[1] - current_x
-        else:
-            return 0, 0
-
-    def _search_prompt(self):
-        import curses
-
-        search_prompt = [TextArgs("Search: ", "green_bold", offset=0)]
-        current_prompt = self.prompt
-        self.prompt = search_prompt
-        with self._store_xy():
-            self._clear_prompt()
-            self.show()
-            curses.echo()
-            expr = self.screen.getstr().decode()
-            curses.noecho()
-            self._update_search_pos(expr)
-            self._parse_text()
-            self._init_pad()
-            # срендерем поверх pad слой с подстветкой
-            self._render_to_pad(self.found_pos)
-            y_offset, x_offset = self.search_next()
-        self.prompt = current_prompt
-        return y_offset, x_offset
-
-    def _do_commands(self):
-        import curses
-
-        while True:
-            self._clear_prompt()
-            try:
-                ch = self.pad.getch()
-            except KeyboardInterrupt:
-                return "n"
-            max_y, max_x = self.screen.getmaxyx()
-            _, pad_max_x = self.pad.getmaxyx()
-            max_y -= 2  # prompt
-            y_offset = 0
-            x_offset = 0
-            margin = 0
-            y_delta = 0
-            x_delta = 0
-
-            y, x = self.pad.getyx()
-            if ch == ord("q"):
-                return "exit"
-            elif ch in [ord("y"), ord("Y")]:
-                return "y"
-            elif ch in [ord("f"), ord("F")]:
-                return "force-yes"
-            elif ch == ord("a"):
-                if self.alternative_text:
-                    self.text, self.alternative_text = self.alternative_text, self.text
-                self.screen.clear()
-                self._parse_text()
-                self._init_pad()
-            elif ch == ord("d"):
-                if self.debug_prompt.text == "":
-                    self.debug_prompt.text = "init"
-                else:
-                    self.debug_prompt.text = ""
-            elif ch == ord("n"):
-                y_offset, x_offset = self.search_next()
-                margin = 10
-            elif ch == ord("N"):
-                y_offset, x_offset = self.search_next(prev=True)
-                margin = 10
-            elif ch == ord("/"):
-                y_offset, x_offset = self._search_prompt()
-                margin = 10
-            elif ch == curses.KEY_UP:
-                y_offset = -1
-            elif ch == curses.KEY_PPAGE:
-                y_offset = -10
-            elif ch == curses.KEY_HOME:
-                y_offset = -len(self.lines)
-            elif ch == curses.KEY_DOWN:
-                y_offset = 1
-            elif ch == curses.KEY_NPAGE:
-                y_offset = 10
-            elif ch == curses.KEY_END:
-                y_offset = len(self.lines)
-            elif ch == curses.KEY_LEFT:
-                x_offset = -1
-            elif ch == curses.KEY_RIGHT:
-                x_offset = 1
-
-            if y_offset or x_offset:
-                y = max(0, y + y_offset)
-                y = min(self.rows, y)
-                x = max(0, x + x_offset)
-                x = min(self.cols, x)
-
-                y_delta = y - (self.top + max_y - margin)
-                if y_delta > 0:
-                    self.top += y_delta
-                elif (y - margin) < self.top:
-                    self.top = y
-
-                self.top = min(self.top, len(self.lines) - max_y)
-
-                x_delta = x - (self.left + max_x)
-                if x_delta > 0:
-                    self.left += x_delta
-                elif x < self.left:
-                    self.left = x
-
-                x = min(x, pad_max_x - 1)
-                self.pad.move(y, x)
-
-            if self.debug_prompt.text != "":
-                debug_line = "y=%s x=%s, x_delta=%s y_delta=%s top=%s, max_y=%s max_x=%s lines=%s" % \
-                             (y, x, x_delta, y_delta, self.top, max_y, max_x, len(self.lines))
-                self.debug_prompt.text = debug_line
-                self.debug_prompt.color = "green_bold"
-                self.debug_prompt.offset = max_x - len(debug_line) - 1
-
-            if self.debug_prompt.text == "":
-                self.page_position.color = "highlight"
-                self.page_position.text = "line %s/%s" % (y, len(self.lines))
-                self.page_position.offset = max_x - len(self.page_position.text) - 1
-
-            self.show()
-
-    def loop(self):
-        import curses
-
-        res = None
-        old_cursor = None
-        try:
-            self.screen = curses.initscr()
-            self.screen.leaveok(True)
-            self.curses_lines = curses.LINES  # pylint: disable=maybe-no-member
-            curses.start_color()
-            curses.noecho()  # no echo key input
-            curses.cbreak()  # input with no-enter keyed
-            try:
-                old_cursor = curses.curs_set(2)
-            except Exception:
-                pass
-            self._init_colors()
-            self._parse_text()
-            self._init_pad()
-            self.pad.move(0, 0)
-            self.show()
-            res = self._do_commands()
-        except Exception as err:
-            get_logger().exception("%s", err)
-        finally:
-            if old_cursor is not None:
-                curses.curs_set(old_cursor)
-            curses.nocbreak()
-            curses.echo()
-            curses.endwin()
-        return res
-
-
-def init_colors():
-    import curses
-
-    curses.init_pair(1, curses.COLOR_GREEN, curses.COLOR_BLACK)
-    curses.init_pair(2, curses.COLOR_CYAN, curses.COLOR_BLACK)
-    curses.init_pair(3, curses.COLOR_RED, curses.COLOR_BLACK)
-    curses.init_pair(4, curses.COLOR_MAGENTA, curses.COLOR_BLACK)
-    curses.init_pair(5, curses.COLOR_YELLOW, curses.COLOR_BLACK)
-    curses.init_pair(6, curses.COLOR_BLUE, curses.COLOR_WHITE)
-    curses.init_pair(7, curses.COLOR_RED, curses.COLOR_WHITE)
-    curses.init_pair(8, curses.COLOR_BLACK, curses.COLOR_WHITE)
-    curses.init_pair(9, curses.COLOR_CYAN, curses.COLOR_BLUE)
-    return {
-        "green": curses.color_pair(1),
-        "green_bold": curses.color_pair(1) | curses.A_BOLD,
-        "cyan": curses.color_pair(2),
-        "red": curses.color_pair(3),
-        "magenta": curses.color_pair(4),
-        "yellow": curses.color_pair(5),
-        "blue": curses.color_pair(6),
-        "highlight": curses.color_pair(7),
-        None: curses.color_pair(8),
-        "cyan_blue": curses.color_pair(9),
-    }
-
-
 class RulebookQuestionHandler:
     def __init__(self, dialogs):
         self._dialogs = dialogs