annet 0.16.35__tar.gz → 1.0.0__tar.gz

{annet-0.16.35/annet.egg-info → annet-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: annet
-Version: 0.16.35
+Version: 1.0.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT

annet-1.0.0/annet/adapters/fetchers/stub/fetcher.py

@@ -0,0 +1,25 @@
+from annet.deploy import Fetcher
+from annet.connectors import AdapterWithConfig
+from typing import Dict, List, Any
+from annet.storage import Device
+
+
+class StubFetcher(Fetcher, AdapterWithConfig):
+    @classmethod
+    def with_config(cls, **kwargs: Dict[str, Any]) -> Fetcher:
+        return cls(**kwargs)
+
+    async def fetch_packages(self,
+                             devices: list[Device],
+                             processes: int = 1,
+                             max_slots: int = 0,
+                             ) -> tuple[dict[Device, str], dict[Device, Any]]:
+        raise NotImplementedError()
+
+    async def fetch(self,
+                    devices: list[Device],
+                    files_to_download: dict[str, list[str]] | None = None,
+                    processes: int = 1,
+                    max_slots: int = 0,
+                    ):
+        raise NotImplementedError()

{annet-0.16.35 → annet-1.0.0}/annet/api/__init__.py

@@ -22,6 +22,8 @@ from typing import (
 )
 
 import colorama
+import annet.deploy
+import annet.deploy_ui
 import annet.lib
 from annet.annlib import jsontools
 from annet.annlib.netdev.views.hardware import HardwareView
@@ -262,7 +264,7 @@ def patch(args: cli_args.ShowPatchOptions, loader: ann_gen.Loader):
     global live_configs  # pylint: disable=global-statement
     if args.config == "running":
         fetcher = annet.deploy.get_fetcher()
-        live_configs = fetcher.fetch(loader.devices, processes=args.parallel)
+        live_configs = annet.lib.do_async(fetcher.fetch(loader.devices, processes=args.parallel))
     stdin = args.stdin(filter_acl=args.filter_acl, config=args.config)
 
     filterer = filtering.filterer_connector.get()
@@ -567,7 +569,7 @@ class Deployer:
             if not diff_obj:
                 self.empty_diff_hostnames.update(dev.hostname for dev in devices)
             if not self.args.no_ask_deploy:
-                # разобъем список устройств на несколько линий
+                # разобьём список устройств на несколько линий
                 dest_name = ""
                 try:
                     _, term_columns_str = os.popen("stty size", "r").read().split()
@@ -596,18 +598,18 @@ class Deployer:
         return diff_lines
 
     def ask_deploy(self) -> str:
-        return self._ask("y", annet.deploy.AskConfirm(
+        return self._ask("y", annet.deploy_ui.AskConfirm(
             text="\n".join(self.diff_lines()),
             alternative_text="\n".join(self.cmd_lines),
         ))
 
     def ask_rollback(self) -> str:
-        return self._ask("n", annet.deploy.AskConfirm(
+        return self._ask("n", annet.deploy_ui.AskConfirm(
             text="Execute rollback?\n",
             alternative_text="",
         ))
 
-    def _ask(self, default_ans: str, ask: annet.deploy.AskConfirm) -> str:
+    def _ask(self, default_ans: str, ask: annet.deploy_ui.AskConfirm) -> str:
         # если filter_acl из stdin то с ним уже не получится работать как с терминалом
         ans = default_ans
         if not self.args.no_ask_deploy:
@@ -666,11 +668,22 @@ def deploy(
     filterer: Filterer,
     fetcher: Fetcher,
     deploy_driver: DeployDriver,
+) -> ExitCode:
+    return annet.lib.do_async(adeploy(args, loader, deployer, filterer, fetcher, deploy_driver))
+
+
+async def adeploy(
+    args: cli_args.DeployOptions,
+    loader: ann_gen.Loader,
+    deployer: Deployer,
+    filterer: Filterer,
+    fetcher: Fetcher,
+    deploy_driver: DeployDriver,
 ) -> ExitCode:
     """ Сгенерировать конфиг для устройств и задеплоить его """
     ret: ExitCode = 0
     global live_configs  # pylint: disable=global-statement
-    live_configs = fetcher.fetch(devices=loader.devices, processes=args.parallel)
+    live_configs = await fetcher.fetch(devices=loader.devices, processes=args.parallel)
     pool = ann_gen.OldNewParallel(args, loader, filterer)
 
     for res in pool.generated_configs(loader.devices):
@@ -687,7 +700,18 @@ def deploy(
         ans = deployer.ask_deploy()
         if ans != "y":
             return 2 ** 2
-        result = annet.lib.do_async(deploy_driver.bulk_deploy(deploy_cmds, args))
+        progress_bar = None
+        if sys.stdout.isatty() and not args.no_progress:
+            progress_bar = annet.deploy_ui.ProgressBars(odict([(device.fqdn, {}) for device in deploy_cmds]))
+            progress_bar.init()
+            with progress_bar:
+                progress_bar.start_terminal_refresher()
+                result = await deploy_driver.bulk_deploy(deploy_cmds, args, progress_bar=progress_bar)
+                await progress_bar.wait_for_exit()
+            progress_bar.screen.clear()
+            progress_bar.stop_terminal_refresher()
+        else:
+            result = await deploy_driver.bulk_deploy(deploy_cmds, args)
 
     rolled_back = False
     rollback_cmds = {deployer.fqdn_to_device[x]: cc for x, cc in result.original_states.items() if cc}
@@ -695,7 +719,7 @@ def deploy(
         ans = deployer.ask_rollback()
         if rollback_cmds and ans == "y":
             rolled_back = True
-            annet.lib.do_async(deploy_driver.bulk_deploy(rollback_cmds, args))
+            await deploy_driver.bulk_deploy(rollback_cmds, args)
 
     if not args.no_check_diff and not rolled_back:
         deployer.check_diff(result, loader)

annet-1.0.0/annet/deploy.py

@@ -0,0 +1,223 @@
+# pylint: disable=unused-argument
+import abc
+import itertools
+from collections import namedtuple
+from typing import Dict, List, Optional, Any, OrderedDict, Tuple, Type
+
+from contextlog import get_logger
+
+from annet.annlib.command import Command, Question, CommandList
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.annlib.rbparser.deploying import MakeMessageMatcher, Answer
+from annet.cli_args import DeployOptions
+from annet.connectors import Connector, get_connector_from_config
+from annet.rulebook import get_rulebook, deploying
+from annet.storage import Device
+
+
+_DeployResultBase = namedtuple("_DeployResultBase", ("hostnames", "results", "durations", "original_states"))
+
+
+class ProgressBar(abc.ABC):
+    @abc.abstractmethod
+    def set_content(self, tile_name: str, content: str):
+        ...
+
+    @abc.abstractmethod
+    def set_progress(self,
+                     tile_name: str,
+                     iteration: int,
+                     total: int,
+                     prefix: str = "",
+                     suffix: str = "",
+                     fill: str = "",
+                     error: bool = False,
+                     ):
+        ...
+
+    @abc.abstractmethod
+    def set_exception(self, tile_name: str, cmd_exc: str, last_cmd: str, progress_max: int):
+        ...
+
+
+class DeployResult(_DeployResultBase):  # noqa: E302
+    def add_results(self, results: dict[str, tuple[list[str], list[Exception]]]) -> None:
+        for hostname, (excs, result) in results.items():
+            self.hostnames.append(hostname)
+            self.results[hostname] = excs
+            self.durations[hostname] = 0.0
+            self.original_states[hostname] = None
+
+
+class _FetcherConnector(Connector["Fetcher"]):
+    name = "Fetcher"
+    ep_name = "deploy_fetcher"
+    ep_by_group_only = "annet.connectors.fetcher"
+
+    def _get_default(self) -> Type["Fetcher"]:
+        # if entry points are broken, try to use direct import
+        import annet.adapters.fetchers.stub.fetcher as stub_fetcher
+        return stub_fetcher.StubFetcher
+
+
+class _DriverConnector(Connector["DeployDriver"]):
+    name = "DeployDriver"
+    ep_name = "deploy_driver"
+    ep_by_group_only = "annet.connectors.deployer"
+
+    def _get_default(self) -> Type["DeployDriver"]:
+        # if entry points are broken, try to use direct import
+        import annet.adapters.deployers.stub.deployer as stub_deployer
+        return stub_deployer.StubDeployDriver
+
+
+fetcher_connector = _FetcherConnector()
+driver_connector = _DriverConnector()
+
+
+class Fetcher(abc.ABC):
+    @abc.abstractmethod
+    async def fetch_packages(self,
+                             devices: list[Device],
+                             processes: int = 1,
+                             max_slots: int = 0,
+                             ) -> tuple[dict[Device, str], dict[Device, Any]]:
+        pass
+
+    @abc.abstractmethod
+    async def fetch(self,
+                    devices: list[Device],
+                    files_to_download: dict[str, list[str]] | None = None,
+                    processes: int = 1,
+                    max_slots: int = 0,
+                    ):
+        pass
+
+
+def get_fetcher() -> Fetcher:
+    connectors = fetcher_connector.get_all()
+    fetcher, _ = get_connector_from_config("fetcher", connectors)
+    return fetcher
+
+
+class DeployDriver(abc.ABC):
+    @abc.abstractmethod
+    async def bulk_deploy(self, deploy_cmds: dict, args: DeployOptions, progress_bar: ProgressBar | None = None) -> DeployResult:
+        pass
+
+    @abc.abstractmethod
+    def apply_deploy_rulebook(self, hw: HardwareView, cmd_paths, do_finalize=True, do_commit=True):
+        pass
+
+    @abc.abstractmethod
+    def build_configuration_cmdlist(self, hw: HardwareView, do_finalize=True, do_commit=True):
+        pass
+
+    @abc.abstractmethod
+    def build_exit_cmdlist(self, hw):
+        pass
+
+
+def get_deployer() -> DeployDriver:
+    connectors = driver_connector.get_all()
+    deployer, _ = get_connector_from_config("deployer", connectors)
+    return deployer
+
+
+# ===
+def scrub_config(text: str, breed: str) -> str:
+    return text
+
+
+def show_bulk_report(hostnames, results, durations, log_dir):
+    pass
+
+
+class RulebookQuestionHandler:
+    def __init__(self, dialogs):
+        self._dialogs = dialogs
+
+    def __call__(self, dev: Connector, cmd: Command, match_content: bytes):
+        content = match_content.strip()
+        content = content.decode()
+        for matcher, answer in self._dialogs.items():
+            if matcher(content):
+                return Command(answer.text)
+
+        get_logger().info("no answer in rulebook. dialogs=%s match_content=%s", self._dialogs, match_content)
+        return None
+
+
+def rb_question_to_question(q: MakeMessageMatcher, a: Answer) -> Question:  # TODO: drop MakeMessageMatcher
+    if not a.send_nl:
+        raise Exception("not supported false send_nl")
+    text: str = q._text  # pylint: disable=protected-access
+    is_regexp = False
+    if text.startswith("/") and text.endswith("/"):
+        is_regexp = True
+        text = text[1:-1]
+    res = Question(question=text, answer=a.text, is_regexp=is_regexp)
+    return res
+
+
+def make_cmd_params(rule: Dict[str, Any]) -> Dict[str, Any]:
+    if rule:
+        qa_handler = RulebookQuestionHandler(rule["attrs"]["dialogs"])
+        qa_list: List[Question] = []
+        for matcher, answer in qa_handler._dialogs.items():  # pylint: disable=protected-access
+            qa_list.append(rb_question_to_question(matcher, answer))
+        return {
+            "questions": qa_list,
+            "timeout": rule["attrs"]["timeout"],
+        }
+    return {
+        "timeout": 30,
+    }
+
+
+def make_apply_commands(rule: dict, hw: HardwareView, do_commit: bool, do_finalize: bool, path: Optional[str] = None):
+    apply_logic = rule["attrs"]["apply_logic"]
+    before, after = apply_logic(hw, do_commit=do_commit, do_finalize=do_finalize, path=path)
+    return before, after
+
+
+def fill_cmd_params(rules: OrderedDict, cmd: Command):
+    rule = deploying.match_deploy_rule(rules, (cmd.cmd,), {})
+    if rule:
+        cmd_params = make_cmd_params(rule)
+        cmd.questions = cmd_params.get("questions", None)
+        cmd.timeout = cmd_params["timeout"]
+
+
+def apply_deploy_rulebook(hw: HardwareView, cmd_paths, do_finalize=True, do_commit=True):
+    rules = get_rulebook(hw)["deploying"]
+    cmds_with_apply = []
+    for cmd_path, context in cmd_paths.items():
+        rule = deploying.match_deploy_rule(rules, cmd_path, context)
+        cmd_params = make_cmd_params(rule)
+        before, after = make_apply_commands(rule, hw, do_commit, do_finalize)
+
+        cmd = Command(cmd_path[-1], **cmd_params)
+        # XXX более чистый способ передавать-мета инфу о команде
+        cmd.level = len(cmd_path) - 1
+        cmds_with_apply.append((cmd, before, after))
+
+    def _key(item):
+        _cmd, before, after = item
+        return (tuple(cmd.cmd for cmd in before), tuple(cmd.cmd for cmd in after))
+
+    cmdlist = CommandList()
+    for _k, cmd_before_after in itertools.groupby(cmds_with_apply, key=_key):
+        cmd_before_after = list(cmd_before_after)
+        _, before, after = cmd_before_after[0]
+        for c in before:
+            c.level = 0
+            fill_cmd_params(rules, c)
+            cmdlist.add_cmd(c)
+        for cmd, _before, _after in cmd_before_after:
+            cmdlist.add_cmd(cmd)
+        for c in after:
+            c.level = 0
+            fill_cmd_params(rules, c)
+            cmdlist.add_cmd(c)
+    return cmdlist